SherbyNet - /page/wikibook/entry.mu

<!DOCTYPE html>

<title>Softmax function</title>

</head>

Softmax function

</h1>

</header>

/* start https://en.wikipedia.org/ */

.mw-parser-output .hatnote{font-style:italic}.mw-parser-output div.hatnote{padding-left:1.6em;margin-bottom:0.5em}.mw-parser-output .hatnote i{font-style:normal}.mw-parser-output .hatnote+link+.hatnote{margin-top:-0.5em}@media print{body.ns-0 .mw-parser-output .hatnote{display:none!important}}

/* end https://en.wikipedia.org/ */

</style><div role="note" class="hatnote navigation-not-searchable">This article is about the smooth approximation of one-hot arg max. For the smooth approximation of max, see <a href="LogSumExp" title="LogSumExp">LogSumExp</a>.</div>

<div role="note" class="hatnote navigation-not-searchable">"Softmax" redirects here. For the Korean video game company, see <a href="ESA_(company)" title="ESA (company)">ESA (company)</a>.</div>

/* start https://en.wikipedia.org/ */

.mw-parser-output .hlist dl,.mw-parser-output .hlist ol,.mw-parser-output .hlist ul{margin:0;padding:0}.mw-parser-output .hlist dd,.mw-parser-output .hlist dt,.mw-parser-output .hlist li{margin:0;display:inline}.mw-parser-output .hlist.inline,.mw-parser-output .hlist.inline dl,.mw-parser-output .hlist.inline ol,.mw-parser-output .hlist.inline ul,.mw-parser-output .hlist dl dl,.mw-parser-output .hlist dl ol,.mw-parser-output .hlist dl ul,.mw-parser-output .hlist ol dl,.mw-parser-output .hlist ol ol,.mw-parser-output .hlist ol ul,.mw-parser-output .hlist ul dl,.mw-parser-output .hlist ul ol,.mw-parser-output .hlist ul ul{display:inline}.mw-parser-output .hlist .mw-empty-li{display:none}.mw-parser-output .hlist dt::after{content:": "}.mw-parser-output .hlist dd::after,.mw-parser-output .hlist li::after{content:" · ";font-weight:bold}.mw-parser-output .hlist dd:last-child::after,.mw-parser-output .hlist dt:last-child::after,.mw-parser-output .hlist li:last-child::after{content:none}.mw-parser-output .hlist dd dd:first-child::before,.mw-parser-output .hlist dd dt:first-child::before,.mw-parser-output .hlist dd li:first-child::before,.mw-parser-output .hlist dt dd:first-child::before,.mw-parser-output .hlist dt dt:first-child::before,.mw-parser-output .hlist dt li:first-child::before,.mw-parser-output .hlist li dd:first-child::before,.mw-parser-output .hlist li dt:first-child::before,.mw-parser-output .hlist li li:first-child::before{content:" (";font-weight:normal}.mw-parser-output .hlist dd dd:last-child::after,.mw-parser-output .hlist dd dt:last-child::after,.mw-parser-output .hlist dd li:last-child::after,.mw-parser-output .hlist dt dd:last-child::after,.mw-parser-output .hlist dt dt:last-child::after,.mw-parser-output .hlist dt li:last-child::after,.mw-parser-output .hlist li dd:last-child::after,.mw-parser-output .hlist li dt:last-child::after,.mw-parser-output .hlist li li:last-child::after{content:")";font-weight:normal}.mw-parser-output .hlist ol{counter-reset:listitem}.mw-parser-output .hlist ol>li{counter-increment:listitem}.mw-parser-output .hlist ol>li::before{content:" "counter(listitem)"\a0 "}.mw-parser-output .hlist dd ol>li:first-child::before,.mw-parser-output .hlist dt ol>li:first-child::before,.mw-parser-output .hlist li ol>li:first-child::before{content:" ("counter(listitem)"\a0 "}

/* end https://en.wikipedia.org/ */

</style><style data-mw-deduplicate="TemplateStyles:r1246091330">

/* start https://en.wikipedia.org/ */

.mw-parser-output .sidebar{width:22em;float:right;clear:right;margin:0.5em 0 1em 1em;background:var(--background-color-neutral-subtle,#f8f9fa);border:1px solid var(--border-color-base,#a2a9b1);padding:0.2em;text-align:center;line-height:1.4em;font-size:88%;border-collapse:collapse;display:table}body.skin-minerva .mw-parser-output .sidebar{display:table!important;float:right!important;margin:0.5em 0 1em 1em!important}.mw-parser-output .sidebar-subgroup{width:100%;margin:0;border-spacing:0}.mw-parser-output .sidebar-left{float:left;clear:left;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-none{float:none;clear:both;margin:0.5em 1em 1em 0}.mw-parser-output .sidebar-outer-title{padding:0 0.4em 0.2em;font-size:125%;line-height:1.2em;font-weight:bold}.mw-parser-output .sidebar-top-image{padding:0.4em}.mw-parser-output .sidebar-top-caption,.mw-parser-output .sidebar-pretitle-with-top-image,.mw-parser-output .sidebar-caption{padding:0.2em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-pretitle{padding:0.4em 0.4em 0;line-height:1.2em}.mw-parser-output .sidebar-title,.mw-parser-output .sidebar-title-with-pretitle{padding:0.2em 0.8em;font-size:145%;line-height:1.2em}.mw-parser-output .sidebar-title-with-pretitle{padding:0.1em 0.4em}.mw-parser-output .sidebar-image{padding:0.2em 0.4em 0.4em}.mw-parser-output .sidebar-heading{padding:0.1em 0.4em}.mw-parser-output .sidebar-content{padding:0 0.5em 0.4em}.mw-parser-output .sidebar-content-with-subgroup{padding:0.1em 0.4em 0.2em}.mw-parser-output .sidebar-above,.mw-parser-output .sidebar-below{padding:0.3em 0.8em;font-weight:bold}.mw-parser-output .sidebar-collapse .sidebar-above,.mw-parser-output .sidebar-collapse .sidebar-below{border-top:1px solid #aaa;border-bottom:1px solid #aaa}.mw-parser-output .sidebar-navbar{text-align:right;font-size:115%;padding:0 0.4em 0.4em}.mw-parser-output .sidebar-list-title{padding:0 0.4em;text-align:left;font-weight:bold;line-height:1.6em;font-size:105%}.mw-parser-output .sidebar-list-title-c{padding:0 0.4em;text-align:center;margin:0 3.3em}@media(max-width:640px){body.mediawiki .mw-parser-output .sidebar{width:100%!important;clear:both;float:none!important;margin-left:0!important;margin-right:0!important}}body.skin--responsive .mw-parser-output .sidebar a>img{max-width:none!important}@media screen{html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-night .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-list-title,html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle{background:transparent!important}html.skin-theme-clientpref-os .mw-parser-output .sidebar:not(.notheme) .sidebar-title-with-pretitle a{color:var(--color-progressive)!important}}@media print{body.ns-0 .mw-parser-output .sidebar{display:none!important}}

/* end https://en.wikipedia.org/ */

</style><style data-mw-deduplicate="TemplateStyles:r886047488">

/* start https://en.wikipedia.org/ */

.mw-parser-output .nobold{font-weight:normal}

/* end https://en.wikipedia.org/ */

</style><table class="sidebar sidebar-collapse nomobile nowraplinks"><tbody><tr><td class="sidebar-pretitle">Part of a series on</td></tr><tr><th class="sidebar-title-with-pretitle"><a href="Machine_learning" title="Machine learning">Machine learning</a> and <a href="Data_mining" title="Data mining">data mining</a></th></tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Paradigms</div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Supervised_learning" title="Supervised learning">Supervised learning</a></li>

<li><a href="Unsupervised_learning" title="Unsupervised learning">Unsupervised learning</a></li>

<li><a href="Semi-supervised_learning" class="mw-redirect" title="Semi-supervised learning">Semi-supervised learning</a></li>

<li><a href="Self-supervised_learning" title="Self-supervised learning">Self-supervised learning</a></li>

<li><a href="Reinforcement_learning" title="Reinforcement learning">Reinforcement learning</a></li>

<li><a href="Meta-learning_(computer_science)" title="Meta-learning (computer science)">Meta-learning</a></li>

<li><a href="Online_machine_learning" title="Online machine learning">Online learning</a></li>

<li><a href="Batch_learning" class="mw-redirect" title="Batch learning">Batch learning</a></li>

<li><a href="Curriculum_learning" title="Curriculum learning">Curriculum learning</a></li>

<li><a href="Rule-based_machine_learning" title="Rule-based machine learning">Rule-based learning</a></li>

<li><a href="Neuro-symbolic_AI" title="Neuro-symbolic AI">Neuro-symbolic AI</a></li>

<li><a href="Neuromorphic_engineering" class="mw-redirect" title="Neuromorphic engineering">Neuromorphic engineering</a></li>

<li><a href="Quantum_machine_learning" title="Quantum machine learning">Quantum machine learning</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Problems</div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Statistical_classification" title="Statistical classification">Classification</a></li>

<li><a href="Generative_model" title="Generative model">Generative modeling</a></li>

<li><a href="Regression_analysis" title="Regression analysis">Regression</a></li>

<li><a href="Cluster_analysis" title="Cluster analysis">Clustering</a></li>

<li><a href="Dimensionality_reduction" title="Dimensionality reduction">Dimensionality reduction</a></li>

<li><a href="Density_estimation" title="Density estimation">Density estimation</a></li>

<li><a href="Anomaly_detection" title="Anomaly detection">Anomaly detection</a></li>

<li><a href="Data_cleaning" class="mw-redirect" title="Data cleaning">Data cleaning</a></li>

<li><a href="Automated_machine_learning" title="Automated machine learning">AutoML</a></li>

<li><a href="Association_rule_learning" title="Association rule learning">Association rules</a></li>

<li><a href="Semantic_analysis_(machine_learning)" title="Semantic analysis (machine learning)">Semantic analysis</a></li>

<li><a href="Structured_prediction" title="Structured prediction">Structured prediction</a></li>

<li><a href="Feature_engineering" title="Feature engineering">Feature engineering</a></li>

<li><a href="Feature_learning" title="Feature learning">Feature learning</a></li>

<li><a href="Learning_to_rank" title="Learning to rank">Learning to rank</a></li>

<li><a href="Grammar_induction" title="Grammar induction">Grammar induction</a></li>

<li><a href="Ontology_learning" title="Ontology learning">Ontology learning</a></li>

<li><a href="Multimodal_learning" title="Multimodal learning">Multimodal learning</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><div style="display: inline-block; line-height: 1.2em; padding: .1em 0;"><a href="Supervised_learning" title="Supervised learning">Supervised learning</a> (<a href="Statistical_classification" title="Statistical classification">classification</a> • <a href="Regression_analysis" title="Regression analysis">regression</a>) </div></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Apprenticeship_learning" title="Apprenticeship learning">Apprenticeship learning</a></li>

<li><a href="Decision_tree_learning" title="Decision tree learning">Decision trees</a></li>

<li><a href="Ensemble_learning" title="Ensemble learning">Ensembles</a>

<ul><li><a href="Bootstrap_aggregating" title="Bootstrap aggregating">Bagging</a></li>

<li><a href="Boosting_(machine_learning)" title="Boosting (machine learning)">Boosting</a></li>

<li><a href="Random_forest" title="Random forest">Random forest</a></li></ul></li>

<li><a href="Linear_regression" title="Linear regression">Linear regression</a></li>

<li><a href="Naive_Bayes_classifier" title="Naive Bayes classifier">Naive Bayes</a></li>

<li><a href="Artificial_neural_network" class="mw-redirect" title="Artificial neural network">Artificial neural networks</a></li>

<li><a href="Logistic_regression" title="Logistic regression">Logistic regression</a></li>

<li><a href="Perceptron" title="Perceptron">Perceptron</a></li>

<li><a href="Relevance_vector_machine" title="Relevance vector machine">Relevance vector machine (RVM)</a></li>

<li><a href="Support_vector_machine" title="Support vector machine">Support vector machine (SVM)</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Cluster_analysis" title="Cluster analysis">Clustering</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="BIRCH" title="BIRCH">BIRCH</a></li>

<li><a href="Hierarchical_clustering" title="Hierarchical clustering">Hierarchical</a></li>

<li><a href="K-means_clustering" title="K-means clustering">k-means</a></li>

<li><a href="Fuzzy_clustering" title="Fuzzy clustering">Fuzzy</a></li>

<li><a href="Expectation%E2%80%93maximization_algorithm" title="Expectation–maximization algorithm">Expectation–maximization (EM)</a></li>

<li> <a href="DBSCAN" title="DBSCAN">DBSCAN</a></li>

<li><a href="OPTICS_algorithm" title="OPTICS algorithm">OPTICS</a></li>

<li><a href="Mean_shift" title="Mean shift">Mean shift</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Dimensionality_reduction" title="Dimensionality reduction">Dimensionality reduction</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Factor_analysis" title="Factor analysis">Factor analysis</a></li>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Structured_prediction" title="Structured prediction">Structured prediction</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Graphical_model" title="Graphical model">Graphical models</a>

<ul><li><a href="Bayesian_network" title="Bayesian network">Bayes net</a></li>

<li><a href="Conditional_random_field" title="Conditional random field">Conditional random field</a></li>

<li><a href="Hidden_Markov_model" title="Hidden Markov model">Hidden Markov</a></li></ul></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Anomaly_detection" title="Anomaly detection">Anomaly detection</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Random_sample_consensus" title="Random sample consensus">RANSAC</a></li>

<li><a href="Local_outlier_factor" title="Local outlier factor">Local outlier factor</a></li>

<li><a href="Isolation_forest" title="Isolation forest">Isolation forest</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Neural_network_(machine_learning)" title="Neural network (machine learning)">Neural networks</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Autoencoder" title="Autoencoder">Autoencoder</a></li>

<li><a href="Deep_learning" title="Deep learning">Deep learning</a></li>

<li><a href="Feedforward_neural_network" title="Feedforward neural network">Feedforward neural network</a></li>

<li><a href="Recurrent_neural_network" title="Recurrent neural network">Recurrent neural network</a>

<li><a href="Reservoir_computing" title="Reservoir computing">reservoir computing</a></li></ul></li>

<li><a href="Boltzmann_machine" title="Boltzmann machine">Boltzmann machine</a>

<ul><li><a href="Restricted_Boltzmann_machine" title="Restricted Boltzmann machine">Restricted</a></li></ul></li>

<li><a href="Diffusion_model" title="Diffusion model">Diffusion model</a></li>

<li><a href="Convolutional_neural_network" title="Convolutional neural network">Convolutional neural network</a>

<li><a href="LeNet" title="LeNet">LeNet</a></li>

<li><a href="AlexNet" title="AlexNet">AlexNet</a></li>

<li><a href="DeepDream" title="DeepDream">DeepDream</a></li></ul></li>

<li><a href="Neural_field" title="Neural field">Neural field</a>

<ul><li><a href="Neural_radiance_field" title="Neural radiance field">Neural radiance field</a></li>

<li><a href="Physics-informed_neural_networks" title="Physics-informed neural networks">Physics-informed neural networks</a></li></ul></li>

<li><a href="Transformer_(deep_learning_architecture)" title="Transformer (deep learning architecture)">Transformer</a>

<ul><li><a href="Vision_transformer" title="Vision transformer">Vision</a></li></ul></li>

<li><a href="Mamba_(deep_learning_architecture)" title="Mamba (deep learning architecture)">Mamba</a></li>

<li><a href="Spiking_neural_network" title="Spiking neural network">Spiking neural network</a></li>

<li><a href="Memtransistor" title="Memtransistor">Memtransistor</a></li>

<li><a href="Electrochemical_RAM" title="Electrochemical RAM">Electrochemical RAM</a> (ECRAM)</li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)"><a href="Reinforcement_learning" title="Reinforcement learning">Reinforcement learning</a></div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Q-learning" title="Q-learning">Q-learning</a></li>

<li><a href="Policy_gradient_method" title="Policy gradient method">Policy gradient</a></li>

<li><a href="State%E2%80%93action%E2%80%93reward%E2%80%93state%E2%80%93action" title="State–action–reward–state–action">SARSA</a></li>

<li><a href="Temporal_difference_learning" title="Temporal difference learning">Temporal difference (TD)</a></li>

<li><a href="Multi-agent_reinforcement_learning" title="Multi-agent reinforcement learning">Multi-agent</a>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Learning with humans</div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Active_learning_(machine_learning)" title="Active learning (machine learning)">Active learning</a></li>

<li><a href="Crowdsourcing" title="Crowdsourcing">Crowdsourcing</a></li>

<li><a href="Human-in-the-loop" title="Human-in-the-loop">Human-in-the-loop</a></li>

<li><a href="Mechanistic_interpretability" title="Mechanistic interpretability">Mechanistic interpretability</a></li>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Model diagnostics</div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Coefficient_of_determination" title="Coefficient of determination">Coefficient of determination</a></li>

<li><a href="Confusion_matrix" title="Confusion matrix">Confusion matrix</a></li>

<li><a href="Learning_curve_(machine_learning)" title="Learning curve (machine learning)">Learning curve</a></li>

<li><a href="Receiver_operating_characteristic" title="Receiver operating characteristic">ROC curve</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Mathematical foundations</div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Kernel_machines" class="mw-redirect" title="Kernel machines">Kernel machines</a></li>

<li><a href="Bias%E2%80%93variance_tradeoff" title="Bias–variance tradeoff">Bias–variance tradeoff</a></li>

<li><a href="Computational_learning_theory" title="Computational learning theory">Computational learning theory</a></li>

<li><a href="Empirical_risk_minimization" title="Empirical risk minimization">Empirical risk minimization</a></li>

<li><a href="Occam_learning" title="Occam learning">Occam learning</a></li>

<li><a href="Probably_approximately_correct_learning" title="Probably approximately correct learning">PAC learning</a></li>

<li><a href="Statistical_learning_theory" title="Statistical learning theory">Statistical learning</a></li>

<li><a href="Vapnik%E2%80%93Chervonenkis_theory" title="Vapnik–Chervonenkis theory">VC theory</a></li>

<li><a href="Topological_deep_learning" title="Topological deep learning">Topological deep learning</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Journals and conferences</div><div class="sidebar-list-content mw-collapsible-content hlist">

<li><a href="Conference_on_Neural_Information_Processing_Systems" title="Conference on Neural Information Processing Systems">NeurIPS</a></li>

<li><a href="International_Joint_Conference_on_Artificial_Intelligence" title="International Joint Conference on Artificial Intelligence">IJCAI</a></li>

</tr><tr><td class="sidebar-content">

<div class="sidebar-list mw-collapsible mw-collapsed machine-learning-list-title"><div class="sidebar-list-title" style="border-top:1px solid #aaa; text-align:center;;color: var(--color-base)">Related articles</div><div class="sidebar-list-content mw-collapsible-content hlist">

<ul><li><a href="Glossary_of_artificial_intelligence" title="Glossary of artificial intelligence">Glossary of artificial intelligence</a></li>

<li><a href="List_of_datasets_for_machine-learning_research" title="List of datasets for machine-learning research">List of datasets for machine-learning research</a>

<ul><li><a href="List_of_datasets_in_computer_vision_and_image_processing" title="List of datasets in computer vision and image processing">List of datasets in computer vision and image processing</a></li></ul></li>

<li><a href="Outline_of_machine_learning" title="Outline of machine learning">Outline of machine learning</a></li></ul></div></div></td>

</tr><tr><td class="sidebar-navbar"><style data-mw-deduplicate="TemplateStyles:r1239400231">

/* start https://en.wikipedia.org/ */

.mw-parser-output .navbar{display:inline;font-size:88%;font-weight:normal}.mw-parser-output .navbar-collapse{float:left;text-align:left}.mw-parser-output .navbar-boxtext{word-spacing:0}.mw-parser-output .navbar ul{display:inline-block;white-space:nowrap;line-height:inherit}.mw-parser-output .navbar-brackets::before{margin-right:-0.125em;content:"[ "}.mw-parser-output .navbar-brackets::after{margin-left:-0.125em;content:" ]"}.mw-parser-output .navbar li{word-spacing:-0.125em}.mw-parser-output .navbar a>span,.mw-parser-output .navbar a>abbr{text-decoration:inherit}.mw-parser-output .navbar-mini abbr{font-variant:small-caps;border-bottom:none;text-decoration:none;cursor:inherit}.mw-parser-output .navbar-ct-full{font-size:114%;margin:0 7em}.mw-parser-output .navbar-ct-mini{font-size:114%;margin:0 4em}html.skin-theme-clientpref-night .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}@media(prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .navbar li a abbr{color:var(--color-base)!important}}@media print{.mw-parser-output .navbar{display:none!important}}

/* end https://en.wikipedia.org/ */

</style></td></tr></tbody></table>

The softmax function, also known as softargmax<a href="#cite_note-1">[1]</a>: 184 or normalized exponential function,<a href="#cite_note-bishop-2">[2]</a>: 198 converts a <a href="Tuple" title="Tuple">tuple</a> of K real numbers into a <a href="Probability_distribution" title="Probability distribution">probability distribution</a> of K possible outcomes. It is a generalization of the <a href="Logistic_function" title="Logistic function">logistic function</a> to multiple dimensions, and is used in <a href="Multinomial_logistic_regression" title="Multinomial logistic regression">multinomial logistic regression</a>. The softmax function is often used as the last <a href="Activation_function" title="Activation function">activation function</a> of a <a href="Artificial_neural_network" class="mw-redirect" title="Artificial neural network">neural network</a> to normalize the output of a network to a <a href="Probability_distribution" title="Probability distribution">probability distribution</a> over predicted output classes.

<div class="mw-heading mw-heading2"><h2 id="Definition">Definition</h2></div>

The softmax function takes as input a tuple z of K real numbers, and normalizes it into a <a href="Probability_distribution" title="Probability distribution">probability distribution</a> consisting of K probabilities proportional to the exponentials of the input numbers. That is, prior to applying softmax, some tuple components could be negative, or greater than one; and might not sum to 1; but after applying softmax, each component will be in the <a href="Interval_(mathematics)" title="Interval (mathematics)">interval</a> <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (0,1)}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle (0,1)}</annotation>

</semantics>

</math><img src="./c79c6838e423c1ed3c7ea532a56dc9f9dae8290b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.168ex; height:2.843ex;" alt="{\displaystyle (0,1)}" loading="lazy">, and the components will add up to 1, so that they can be interpreted as probabilities. Furthermore, the larger input components will correspond to larger probabilities.

Formally, the standard (unit) softmax function <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sigma \colon \mathbb {R} ^{K}\to (0,1)^{K}}">

<msup>

</mrow>

</mrow>

</msup>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sigma \colon \mathbb {R} ^{K}\to (0,1)^{K}}</annotation>

</semantics>

</math><img src="./5309c884cbe9d4c1aab1134d2e8c730c88376446.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:16.21ex; height:3.176ex;" alt="{\displaystyle \sigma \colon \mathbb {R} ^{K}\to (0,1)^{K}}" loading="lazy">, where ⁠<math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle K>1}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle K>1}</annotation>

</semantics>

</math><img src="./6306c7d6d3bfb26d8a720f8ae7bf0e76e4d71d7d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:6.327ex; height:2.176ex;" alt="{\displaystyle K>1}" loading="lazy">⁠, takes a tuple <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathbf {z} =(z_{1},\dotsc ,z_{K})\in \mathbb {R} ^{K}}">

</mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mathbf {z} =(z_{1},\dotsc ,z_{K})\in \mathbb {R} ^{K}}</annotation>

</semantics>

</math><img src="./cf962e53664c868e65fe35636fd13a0fe40e56ab.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:22.395ex; height:3.176ex;" alt="{\displaystyle \mathbf {z} =(z_{1},\dotsc ,z_{K})\in \mathbb {R} ^{K}}" loading="lazy"> and computes each component of vector <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sigma (\mathbf {z} )\in (0,1)^{K}}">

</mrow>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sigma (\mathbf {z} )\in (0,1)^{K}}</annotation>

</semantics>

</math><img src="./aade360497acd4816db99e74bd651449fcc2eb74.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:14.029ex; height:3.176ex;" alt="{\displaystyle \sigma (\mathbf {z} )\in (0,1)^{K}}" loading="lazy"> with

</mrow>

<msub>

</mrow>

</msub>

<mfrac>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

<mrow>

</mrow>

</mrow>

</munderover>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sigma (\mathbf {z} )_{i}={\frac {e^{z_{i}}}{\sum _{j=1}^{K}e^{z_{j}}}}\,.}</annotation>

</semantics>

</math>

In words, the softmax applies the standard <a href="Exponential_function" title="Exponential function">exponential function</a> to each element <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle z_{i}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle z_{i}}</annotation>

</semantics>

</math><img src="./5c6e920bac39ad09fff4efef16254595091a1025.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.881ex; height:2.009ex;" alt="{\displaystyle z_{i}}" loading="lazy"> of the input tuple <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathbf {z} }">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mathbf {z} }</annotation>

</semantics>

</math><img src="./82eca5d0928078d5a61b9e7e98cc73db31070909.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.188ex; height:1.676ex;" alt="{\displaystyle \mathbf {z} }" loading="lazy"> (consisting of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle K}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle K}</annotation>

</semantics>

</math><img src="./2b76fce82a62ed5461908f0dc8f037de4e3686b0.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.066ex; height:2.176ex;" alt="{\displaystyle K}" loading="lazy"> real numbers), and normalizes these values by dividing by the sum of all these exponentials. The normalization ensures that the sum of the components of the output vector <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sigma (\mathbf {z} )}">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sigma (\mathbf {z} )}</annotation>

</semantics>

</math><img src="./2e610a6185b8850a6f567c4902387b17f0ec1652.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:4.327ex; height:2.843ex;" alt="{\displaystyle \sigma (\mathbf {z} )}" loading="lazy"> is 1. The term "softmax" derives from the amplifying effects of the exponential on any maxima in the input tuple. For example, the standard softmax of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (1,2,8)}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle (1,2,8)}</annotation>

</semantics>

</math><img src="./8ee1ed5d3173bc3d1b616d1c628f53fc172498b2.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:7.365ex; height:2.843ex;" alt="{\displaystyle (1,2,8)}" loading="lazy"> is approximately <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (0.001,0.002,0.997)}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle (0.001,0.002,0.997)}</annotation>

</semantics>

</math><img src="./9801d1f56ccabff60706edfb09cc4fed15205df0.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:19.767ex; height:2.843ex;" alt="{\displaystyle (0.001,0.002,0.997)}" loading="lazy">, which amounts to assigning almost all of the total unit weight in the result to the position of the tuple's maximal element (of 8).

In general, instead of e a different <a href="Base_(exponentiation)" class="mw-redirect" title="Base (exponentiation)">base</a> b > 0 can be used. As above, if b > 1 then larger input components will result in larger output probabilities, and increasing the value of b will create probability distributions that are more concentrated around the positions of the largest input values. Conversely, if 0 then smaller input components will result in larger output probabilities, and decreasing the value of b will create probability distributions that are more concentrated around the positions of the smallest input values. Writing <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle b=e^{\beta }}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle b=e^{\beta }}</annotation>

</semantics>

</math><img src="./d879d4c1464e135ae07c282212a5c8812dfbef04.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:6.354ex; height:2.676ex;" alt="{\displaystyle b=e^{\beta }}" loading="lazy"> or <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle b=e^{-\beta }}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle b=e^{-\beta }}</annotation>

</semantics>

</math><img src="./a9497d7e7769fa17a0ae348b7fc02218ca8c3ded.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:7.632ex; height:2.676ex;" alt="{\displaystyle b=e^{-\beta }}" loading="lazy"><a href="#cite_note-3">[a]</a> (for real β)<a href="#cite_note-4">[b]</a> yields the expressions:<a href="#cite_note-5">[c]</a>

</mrow>

<msub>

</mrow>

</msub>

<mfrac>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

<mrow>

</mrow>

</mrow>

</munderover>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mrow>

</mrow>

<msub>

</mrow>

</msub>

<mfrac>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

<mrow>

</mrow>

</mrow>

</munderover>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sigma (\mathbf {z} )_{i}={\frac {e^{\beta z_{i}}}{\sum _{j=1}^{K}e^{\beta z_{j}}}}{\text{ or }}\sigma (\mathbf {z} )_{i}={\frac {e^{-\beta z_{i}}}{\sum _{j=1}^{K}e^{-\beta z_{j}}}}{\text{ for }}i=1,\dotsc ,K.}</annotation>

</semantics>

</math>

A value proportional to the reciprocal of β is sometimes referred to as the temperature: <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\textstyle \beta =1/kT}">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\textstyle \beta =1/kT}</annotation>

</semantics>

</math><img src="./73542ddcdcabcac5dbf6f9fca57547b529a69112.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:9.603ex; height:2.843ex;" alt="{\textstyle \beta =1/kT}" loading="lazy">, where k is typically 1 or the <a href="Boltzmann_constant" title="Boltzmann constant">Boltzmann constant</a> and T is the temperature. A higher temperature results in a more uniform output distribution (i.e. with higher <a href="Entropy_(information_theory)" title="Entropy (information theory)">entropy</a>; it is "more random"), while a lower temperature results in a sharper output distribution, with one value dominating.

In some fields, the base is fixed, corresponding to a fixed scale,<a href="#cite_note-6">[d]</a> while in others the parameter β (or T) is varied.

<div class="mw-heading mw-heading2"><h2 id="Interpretations">Interpretations</h2></div>

<div class="mw-heading mw-heading3"><h3 id="Smooth_arg_max">Smooth arg max</h3></div>

The Softmax function is a smooth approximation to the <a href="Arg_max" title="Arg max">arg max</a> function: the function whose value is the index of a tuple's largest element. The name "softmax" may be misleading. Softmax is not a <a href="Smooth_maximum" title="Smooth maximum">smooth maximum</a> (that is, a <a href="Smooth_approximation" class="mw-redirect" title="Smooth approximation">smooth approximation</a> to the <a href="Maximum" class="mw-redirect" title="Maximum">maximum</a> function). The term "softmax" is also used for the closely related <a href="LogSumExp" title="LogSumExp">LogSumExp</a> function, which is a smooth maximum. For this reason, some prefer the more accurate term "softargmax", though the term "softmax" is conventional in machine learning.<a href="#cite_note-sako2018-7">[3]</a><a href="#cite_note-FOOTNOTEGoodfellowBengioCourville2016183–184-8">[4]</a> This section uses the term "softargmax" for clarity.

Formally, instead of considering the arg max as a function with categorical output <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle 1,\dots ,n}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle 1,\dots ,n}</annotation>

</semantics>

</math><img src="./3183c8ced500768421aa5039ac010066d8122ae3.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:7.735ex; height:2.509ex;" alt="{\displaystyle 1,\dots ,n}" loading="lazy"> (corresponding to the index), consider the arg max function with <a href="One-hot" title="One-hot">one-hot</a> representation of the output (assuming there is a unique maximum arg):

</mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \operatorname {arg\,max} (z_{1},\,\dots ,\,z_{n})=(y_{1},\,\dots ,\,y_{n})=(0,\,\dots ,\,0,\,1,\,0,\,\dots ,\,0),}</annotation>

</semantics>

</math>

where the output coordinate <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle y_{i}=1}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle y_{i}=1}</annotation>

</semantics>

</math><img src="./13864bfc60110b9306a50b300d2d72b22fa1e371.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:6.2ex; height:2.509ex;" alt="{\displaystyle y_{i}=1}" loading="lazy"> if and only if <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle i}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle i}</annotation>

</semantics>

</math><img src="./add78d8608ad86e54951b8c8bd6c8d8416533d20.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:0.802ex; height:2.176ex;" alt="{\displaystyle i}" loading="lazy"> is the arg max of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (z_{1},\dots ,z_{n})}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle (z_{1},\dots ,z_{n})}</annotation>

</semantics>

</math><img src="./cccc6bc9d572e38cf2534444278f197fbc26574c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:11.423ex; height:2.843ex;" alt="{\displaystyle (z_{1},\dots ,z_{n})}" loading="lazy">, meaning <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle z_{i}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle z_{i}}</annotation>

</semantics>

</math><img src="./5c6e920bac39ad09fff4efef16254595091a1025.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.881ex; height:2.009ex;" alt="{\displaystyle z_{i}}" loading="lazy"> is the unique maximum value of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (z_{1},\,\dots ,\,z_{n})}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle (z_{1},\,\dots ,\,z_{n})}</annotation>

</semantics>

</math><img src="./a512fe2ad79c172f6cd12647ba291a2ae1363284.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:12.197ex; height:2.843ex;" alt="{\displaystyle (z_{1},\,\dots ,\,z_{n})}" loading="lazy">. For example, in this encoding <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {arg\,max} (1,5,10)=(0,0,1),}">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \operatorname {arg\,max} (1,5,10)=(0,0,1),}</annotation>

</semantics>

</math><img src="./676a7db4c489a14e68d36d26339ec95fbeeea4f7.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:27.586ex; height:2.843ex;" alt="{\displaystyle \operatorname {arg\,max} (1,5,10)=(0,0,1),}" loading="lazy"> since the third argument is the maximum.

This can be generalized to multiple arg max values (multiple equal <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle z_{i}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle z_{i}}</annotation>

</semantics>

</math><img src="./5c6e920bac39ad09fff4efef16254595091a1025.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.881ex; height:2.009ex;" alt="{\displaystyle z_{i}}" loading="lazy"> being the maximum) by dividing the 1 between all max args; formally 1/k where k is the number of arguments assuming the maximum. For example, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {arg\,max} (1,\,5,\,5)=(0,\,1/2,\,1/2),}">

</mrow>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \operatorname {arg\,max} (1,\,5,\,5)=(0,\,1/2,\,1/2),}</annotation>

</semantics>

</math><img src="./2c97a4b64c780c7da0451173cd534d56112b647d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:32.622ex; height:2.843ex;" alt="{\displaystyle \operatorname {arg\,max} (1,\,5,\,5)=(0,\,1/2,\,1/2),}" loading="lazy"> since the second and third argument are both the maximum. In case all arguments are equal, this is simply <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {arg\,max} (z,\dots ,z)=(1/n,\dots ,1/n).}">

</mrow>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \operatorname {arg\,max} (z,\dots ,z)=(1/n,\dots ,1/n).}</annotation>

</semantics>

</math><img src="./9cbff8b2dedd0b4e138cc5e0a8e458267668771a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:35.285ex; height:2.843ex;" alt="{\displaystyle \operatorname {arg\,max} (z,\dots ,z)=(1/n,\dots ,1/n).}" loading="lazy"> Points z with multiple arg max values are <a href="Singular_point_of_an_algebraic_variety" title="Singular point of an algebraic variety">singular points</a> (or singularities, and form the singular set) – these are the points where arg max is discontinuous (with a <a href="Jump_discontinuity" class="mw-redirect" title="Jump discontinuity">jump discontinuity</a>) – while points with a single arg max are known as non-singular or regular points.

With the last expression given in the introduction, softargmax is now a smooth approximation of arg max: as ⁠<math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \beta \to \infty }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \beta \to \infty }</annotation>

</semantics>

</math><img src="./b84f8f60e0efae2dcf23c3eeb8830105d881b9d0.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:7.27ex; height:2.509ex;" alt="{\displaystyle \beta \to \infty }" loading="lazy">⁠, softargmax converges to arg max. There are various notions of convergence of a function; softargmax converges to arg max <a href="Pointwise_convergence" title="Pointwise convergence">pointwise</a>, meaning for each fixed input z as ⁠<math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \beta \to \infty }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \beta \to \infty }</annotation>

</semantics>

</math><img src="./b84f8f60e0efae2dcf23c3eeb8830105d881b9d0.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:7.27ex; height:2.509ex;" alt="{\displaystyle \beta \to \infty }" loading="lazy">⁠, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sigma _{\beta }(\mathbf {z} )\to \operatorname {arg\,max} (\mathbf {z} ).}">

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sigma _{\beta }(\mathbf {z} )\to \operatorname {arg\,max} (\mathbf {z} ).}</annotation>

</semantics>

</math><img src="./a2c33a8d1d5a236f6d0dad5c79da1c96c580aa14.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:20.706ex; height:3.009ex;" alt="{\displaystyle \sigma _{\beta }(\mathbf {z} )\to \operatorname {arg\,max} (\mathbf {z} ).}" loading="lazy"> However, softargmax does not <a href="Uniform_convergence" title="Uniform convergence">converge uniformly</a> to arg max, meaning intuitively that different points converge at different rates, and may converge arbitrarily slowly. In fact, softargmax is continuous, but arg max is not continuous at the singular set where two coordinates are equal, while the uniform limit of continuous functions is continuous. The reason it fails to converge uniformly is that for inputs where two coordinates are almost equal (and one is the maximum), the arg max is the index of one or the other, so a small change in input yields a large change in output. For example, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sigma _{\beta }(1,\,1.0001)\to (0,1),}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sigma _{\beta }(1,\,1.0001)\to (0,1),}</annotation>

</semantics>

</math><img src="./e2e6908308a03bc894b870a9cf4a2b7eddbd9866.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:22.782ex; height:3.009ex;" alt="{\displaystyle \sigma _{\beta }(1,\,1.0001)\to (0,1),}" loading="lazy"> but <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sigma _{\beta }(1,\,0.9999)\to (1,\,0),}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sigma _{\beta }(1,\,0.9999)\to (1,\,0),}</annotation>

</semantics>

</math><img src="./d48c70a91851297e78294b8f4c70deebde8e7c81.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:23.17ex; height:3.009ex;" alt="{\displaystyle \sigma _{\beta }(1,\,0.9999)\to (1,\,0),}" loading="lazy"> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sigma _{\beta }(1,\,1)=1/2}">

<msub>

</mrow>

</msub>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sigma _{\beta }(1,\,1)=1/2}</annotation>

</semantics>

</math><img src="./48ac82a615c07956296e61b937f5cc4b38051f14.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:14.643ex; height:3.009ex;" alt="{\displaystyle \sigma _{\beta }(1,\,1)=1/2}" loading="lazy"> for all inputs: the closer the points are to the singular set <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (x,x)}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle (x,x)}</annotation>

</semantics>

</math><img src="./72f9e25892f6d000349b8bb6578a59567efbdd63.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.503ex; height:2.843ex;" alt="{\displaystyle (x,x)}" loading="lazy">, the slower they converge. However, softargmax does <a href="Compact_convergence" title="Compact convergence">converge compactly</a> on the non-singular set.

Conversely, as ⁠<math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \beta \to -\infty }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \beta \to -\infty }</annotation>

</semantics>

</math><img src="./cc1ace11f1a4314b872c0dfb5d09d932f354a74e.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:9.078ex; height:2.509ex;" alt="{\displaystyle \beta \to -\infty }" loading="lazy">⁠, softargmax converges to arg min in the same way, where here the singular set is points with two arg min values. In the language of <a href="Tropical_analysis" title="Tropical analysis">tropical analysis</a>, the softmax is a <a href="Deformation_theory" class="mw-redirect" title="Deformation theory">deformation</a> or "quantization" of arg max and arg min, corresponding to using the <a href="Log_semiring" title="Log semiring">log semiring</a> instead of the <a href="Max-plus_semiring" class="mw-redirect" title="Max-plus semiring">max-plus semiring</a> (respectively <a href="Min-plus_semiring" class="mw-redirect" title="Min-plus semiring">min-plus semiring</a>), and recovering the arg max or arg min by taking the limit is called "tropicalization" or "dequantization".

It is also the case that, for any fixed β, if one input ⁠<math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle z_{i}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle z_{i}}</annotation>

</semantics>

</math><img src="./5c6e920bac39ad09fff4efef16254595091a1025.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.881ex; height:2.009ex;" alt="{\displaystyle z_{i}}" loading="lazy">⁠ is much larger than the others relative to the temperature, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle T=1/\beta }">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle T=1/\beta }</annotation>

</semantics>

</math><img src="./8bfbd66d2e94a4b4ba5a9f542802f9066dc36450.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:8.392ex; height:2.843ex;" alt="{\displaystyle T=1/\beta }" loading="lazy">, the output is approximately the arg max. For example, a difference of 10 is large relative to a temperature of 1:

<msub>

</mrow>

</msub>

<mrow>

<mrow>

</mrow>

<mrow>

<mrow>

<msup>

</mrow>

</msup>

</mrow>

</mrow>

<msup>

</mrow>

</msup>

</mrow>

<mrow>

<mrow>

<msup>

</mrow>

</msup>

</mrow>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sigma (0,\,10):=\sigma _{1}(0,\,10)=\left(1/\left(1+e^{10}\right),\,e^{10}/\left(1+e^{10}\right)\right)\approx (0.00005,\,0.99995)}</annotation>

</semantics>

</math>

However, if the difference is small relative to the temperature, the value is not close to the arg max. For example, a difference of 10 is small relative to a temperature of 100:

<msub>

</mrow>

</mrow>

</msub>

<mrow>

<mrow>

</mrow>

<mrow>

<mrow>

<msup>

</mrow>

</mrow>

</msup>

</mrow>

</mrow>

<msup>

</mrow>

</mrow>

</msup>

</mrow>

<mrow>

<mrow>

<msup>

</mrow>

</mrow>

</msup>

</mrow>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sigma _{1/100}(0,\,10)=\left(1/\left(1+e^{1/10}\right),\,e^{1/10}/\left(1+e^{1/10}\right)\right)\approx (0.475,\,0.525).}</annotation>

</semantics>

</math>

As ⁠<math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \beta \to \infty }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \beta \to \infty }</annotation>

</semantics>

</math><img src="./b84f8f60e0efae2dcf23c3eeb8830105d881b9d0.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:7.27ex; height:2.509ex;" alt="{\displaystyle \beta \to \infty }" loading="lazy">⁠, temperature goes to zero, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle T=1/\beta \to 0}">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle T=1/\beta \to 0}</annotation>

</semantics>

</math><img src="./9f9b2ce8c075bb122f6fc3cbebfc5ac0efe75a0d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:13.168ex; height:2.843ex;" alt="{\displaystyle T=1/\beta \to 0}" loading="lazy">, so eventually all differences become large (relative to a shrinking temperature), which gives another interpretation for the limit behavior.

<div class="mw-heading mw-heading3"><h3 id="Statistical_mechanics">Statistical mechanics</h3></div>

In <a href="Statistical_mechanics" title="Statistical mechanics">statistical mechanics</a>, the softargmax function is known as the <a href="Boltzmann_distribution" title="Boltzmann distribution">Boltzmann distribution</a> (or <a href="Gibbs_distribution" class="mw-redirect" title="Gibbs distribution">Gibbs distribution</a>):<a href="#cite_note-9">[5]</a>: 7 the index set <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {1,\,\dots ,\,k}}">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {1,\,\dots ,\,k}}</annotation>

</semantics>

</math><img src="./958f8a6502be20d182b4d51708ffd0d6f02169a5.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:8.326ex; height:2.509ex;" alt="{\displaystyle {1,\,\dots ,\,k}}" loading="lazy"> are the <a href="Microstate_(statistical_mechanics)" title="Microstate (statistical mechanics)">microstates</a> of the system; the inputs <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle z_{i}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle z_{i}}</annotation>

</semantics>

</math><img src="./5c6e920bac39ad09fff4efef16254595091a1025.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.881ex; height:2.009ex;" alt="{\displaystyle z_{i}}" loading="lazy"> are the energies of that state; the denominator is known as the <a href="Partition_function_(statistical_mechanics)" title="Partition function (statistical mechanics)">partition function</a>, often denoted by Z; and the factor β is called the <a href="Coldness" class="mw-redirect" title="Coldness">coldness</a> (or <a href="Thermodynamic_beta" title="Thermodynamic beta">thermodynamic beta</a>, or <a href="Inverse_temperature" class="mw-redirect" title="Inverse temperature">inverse temperature</a>).

<div class="mw-heading mw-heading2"><h2 id="Applications">Applications</h2></div>

The softmax function is used in various <a href="Multiclass_classification" title="Multiclass classification">multiclass classification</a> methods, such as <a href="Multinomial_logistic_regression" title="Multinomial logistic regression">multinomial logistic regression</a> (also known as softmax regression),<a href="#cite_note-bishop-2">[2]</a>: 206–209 <a href="#cite_note-10">[6]</a> multiclass <a href="Linear_discriminant_analysis" title="Linear discriminant analysis">linear discriminant analysis</a>, <a href="Naive_Bayes_classifier" title="Naive Bayes classifier">naive Bayes classifiers</a>, and <a href="Artificial_neural_network" class="mw-redirect" title="Artificial neural network">artificial neural networks</a>.<a href="#cite_note-11">[7]</a> Specifically, in multinomial logistic regression and linear discriminant analysis, the input to the function is the result of K distinct <a href="Linear_function" title="Linear function">linear functions</a>, and the predicted probability for the jth class given a sample tuple x and a weighting vector w is:

</mrow>

<mfrac>

<msup>

<msup>

</mrow>

</mrow>

</msup>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

<mrow>

</mrow>

</mrow>

</munderover>

<msup>

<msup>

</mrow>

</mrow>

</msup>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle P(y=j\mid \mathbf {x} )={\frac {e^{\mathbf {x} ^{\mathsf {T}}\mathbf {w} _{j}}}{\sum _{k=1}^{K}e^{\mathbf {x} ^{\mathsf {T}}\mathbf {w} _{k}}}}}</annotation>

</semantics>

</math>

This can be seen as the <a href="Function_composition" title="Function composition">composition</a> of K linear functions <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathbf {x} \mapsto \mathbf {x} ^{\mathsf {T}}\mathbf {w} _{1},\ldots ,\mathbf {x} \mapsto \mathbf {x} ^{\mathsf {T}}\mathbf {w} _{K}}">

</mrow>

<msup>

</mrow>

</mrow>

</msup>

<msub>

</mrow>

</mrow>

</msub>

</mrow>

<msup>

</mrow>

</mrow>

</msup>

<msub>

</mrow>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mathbf {x} \mapsto \mathbf {x} ^{\mathsf {T}}\mathbf {w} _{1},\ldots ,\mathbf {x} \mapsto \mathbf {x} ^{\mathsf {T}}\mathbf {w} _{K}}</annotation>

</semantics>

</math><img src="./933ef5b00ec8c1e9470c87352dca2e9810fb003c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:27.363ex; height:3.009ex;" alt="{\displaystyle \mathbf {x} \mapsto \mathbf {x} ^{\mathsf {T}}\mathbf {w} _{1},\ldots ,\mathbf {x} \mapsto \mathbf {x} ^{\mathsf {T}}\mathbf {w} _{K}}" loading="lazy"> and the softmax function (where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathbf {x} ^{\mathsf {T}}\mathbf {w} }">

<msup>

</mrow>

</mrow>

</msup>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mathbf {x} ^{\mathsf {T}}\mathbf {w} }</annotation>

</semantics>

</math><img src="./ed370c04ba00660e67f31aa882d791a742a0da9c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:4.694ex; height:2.676ex;" alt="{\displaystyle \mathbf {x} ^{\mathsf {T}}\mathbf {w} }" loading="lazy"> denotes the inner product of <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathbf {x} }">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mathbf {x} }</annotation>

</semantics>

</math><img src="./32adf004df5eb0a8c7fd8c0b6b7405183c5a5ef2.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.411ex; height:1.676ex;" alt="{\displaystyle \mathbf {x} }" loading="lazy"> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathbf {w} }">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mathbf {w} }</annotation>

</semantics>

</math><img src="./20795664b5b048744a2fd88977851104cc5816f8.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.931ex; height:1.676ex;" alt="{\displaystyle \mathbf {w} }" loading="lazy">). The operation is equivalent to applying a linear operator defined by <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathbf {w} }">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mathbf {w} }</annotation>

</semantics>

</math><img src="./20795664b5b048744a2fd88977851104cc5816f8.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.931ex; height:1.676ex;" alt="{\displaystyle \mathbf {w} }" loading="lazy"> to tuples <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathbf {x} }">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mathbf {x} }</annotation>

</semantics>

</math><img src="./32adf004df5eb0a8c7fd8c0b6b7405183c5a5ef2.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.411ex; height:1.676ex;" alt="{\displaystyle \mathbf {x} }" loading="lazy">, thus transforming the original, probably highly-dimensional, input to vectors in a K-dimensional space <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathbb {R} ^{K}}">

<msup>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mathbb {R} ^{K}}</annotation>

</semantics>

</math><img src="./a7fe4c8791d7d313b6c2f1ddca68b6662fcaf60a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:3.371ex; height:2.676ex;" alt="{\displaystyle \mathbb {R} ^{K}}" loading="lazy">.

<div class="mw-heading mw-heading3"><h3 id="Neural_networks">Neural networks</h3></div>

The standard softmax function is often used in the final layer of a neural network-based classifier. Such networks are commonly trained under a <a href="Cross_entropy" class="mw-redirect" title="Cross entropy">log loss</a> (or <a href="Cross-entropy" title="Cross-entropy">cross-entropy</a>) regime, giving a non-linear variant of multinomial logistic regression.

Since the function maps a tuple and a specific index <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle i}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle i}</annotation>

</semantics>

</math><img src="./add78d8608ad86e54951b8c8bd6c8d8416533d20.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:0.802ex; height:2.176ex;" alt="{\displaystyle i}" loading="lazy"> to a real value, the derivative needs to take the index into account:

<mfrac>

<mrow>

<msub>

</mrow>

</msub>

</mrow>

</mfrac>

</mrow>

</mrow>

</mrow>

<msub>

</mrow>

</msub>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\frac {\partial }{\partial q_{k}}}\sigma ({\textbf {q}},i)=\sigma ({\textbf {q}},i)(\delta _{ik}-\sigma ({\textbf {q}},k)).}</annotation>

</semantics>

</math>

This expression is symmetrical in the indexes <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle i,k}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle i,k}</annotation>

</semantics>

</math><img src="./c3885ba48fa465d00556c235647088efbd248c26.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:3.048ex; height:2.509ex;" alt="{\displaystyle i,k}" loading="lazy"> and thus may also be expressed as

<mfrac>

<mrow>

<msub>

</mrow>

</msub>

</mrow>

</mfrac>

</mrow>

</mrow>

</mrow>

<msub>

</mrow>

</msub>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\frac {\partial }{\partial q_{k}}}\sigma ({\textbf {q}},i)=\sigma ({\textbf {q}},k)(\delta _{ik}-\sigma ({\textbf {q}},i)).}</annotation>

</semantics>

</math>

Here, the <a href="Kronecker_delta" title="Kronecker delta">Kronecker delta</a> is used for simplicity (cf. the derivative of a <a href="Sigmoid_function" title="Sigmoid function">sigmoid function</a>, being expressed via the function itself).

To ensure stable numerical computations subtracting the maximum value from the input tuple is common. This approach, while not altering the output or the derivative theoretically, enhances stability by directly controlling the maximum exponent value computed.

If the function is scaled with the parameter <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \beta }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \beta }</annotation>

</semantics>

</math><img src="./7ed48a5e36207156fb792fa79d29925d2f7901e8.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.332ex; height:2.509ex;" alt="{\displaystyle \beta }" loading="lazy">, then these expressions must be multiplied by <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \beta }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \beta }</annotation>

</semantics>

</math><img src="./7ed48a5e36207156fb792fa79d29925d2f7901e8.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.332ex; height:2.509ex;" alt="{\displaystyle \beta }" loading="lazy">.

See <a href="Multinomial_logit" class="mw-redirect" title="Multinomial logit">multinomial logit</a> for a probability model which uses the softmax activation function.

<div class="mw-heading mw-heading3"><h3 id="Reinforcement_learning">Reinforcement learning</h3></div>

In the field of <a href="Reinforcement_learning" title="Reinforcement learning">reinforcement learning</a>, a softmax function can be used to convert values into action probabilities. The function commonly used is:<a href="#cite_note-12">[8]</a>

<msub>

</mrow>

</msub>

<mfrac>

<mrow>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

<mrow>

</mrow>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</mfrac>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle P_{t}(a)={\frac {\exp(q_{t}(a)/\tau )}{\sum _{i=1}^{n}\exp(q_{t}(i)/\tau )}}{\text{,}}}</annotation>

</semantics>

</math>

where the action value <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle q_{t}(a)}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle q_{t}(a)}</annotation>

</semantics>

</math><img src="./8e560f478676b4773a3ab72e65106883dd6b2d2b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:4.902ex; height:2.843ex;" alt="{\displaystyle q_{t}(a)}" loading="lazy"> corresponds to the expected reward of following action a and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \tau }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \tau }</annotation>

</semantics>

</math><img src="./38a7dcde9730ef0853809fefc18d88771f95206c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.202ex; height:1.676ex;" alt="{\displaystyle \tau }" loading="lazy"> is called a temperature parameter (in allusion to <a href="Statistical_mechanics" title="Statistical mechanics">statistical mechanics</a>). For high temperatures (<math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \tau \to \infty }">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \tau \to \infty }</annotation>

</semantics>

</math><img src="./f9d637e88503e9b1f8f1029cdd2f6b748f0318f0.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:7.14ex; height:1.843ex;" alt="{\displaystyle \tau \to \infty }" loading="lazy">), all actions have nearly the same probability and the lower the temperature, the more expected rewards affect the probability. For a low temperature (<math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \tau \to 0^{+}}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \tau \to 0^{+}}</annotation>

</semantics>

</math><img src="./9824efb844e5497f64b5ed982054fe3df3aebb70.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:7.489ex; height:2.509ex;" alt="{\displaystyle \tau \to 0^{+}}" loading="lazy">), the probability of the action with the highest expected reward tends to 1.

<div class="mw-heading mw-heading2"><h2 id="Computational_complexity_and_remedies">Computational complexity and remedies</h2></div>

In neural network applications, the number K of possible outcomes is often large, e.g. in case of <a href="Language_model" title="Language model">neural language models</a> that predict the most likely outcome out of a vocabulary which might contain millions of possible words.<a href="#cite_note-:0-13">[9]</a> This can make the calculations for the softmax layer (i.e. the matrix multiplications to determine the <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle z_{i}}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle z_{i}}</annotation>

</semantics>

</math><img src="./5c6e920bac39ad09fff4efef16254595091a1025.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.881ex; height:2.009ex;" alt="{\displaystyle z_{i}}" loading="lazy">, followed by the application of the softmax function itself) computationally expensive.<a href="#cite_note-:0-13">[9]</a><a href="#cite_note-:1-14">[10]</a> What's more, the <a href="Gradient_descent" title="Gradient descent">gradient descent</a> <a href="Backpropagation" title="Backpropagation">backpropagation</a> method for training such a neural network involves calculating the softmax for every training example, and the number of training examples can also become large. The computational effort for the softmax became a major limiting factor in the development of larger neural language models, motivating various remedies to reduce training times.<a href="#cite_note-:0-13">[9]</a><a href="#cite_note-:1-14">[10]</a>

Approaches that reorganize the softmax layer for more efficient calculation include the hierarchical softmax and the differentiated softmax.<a href="#cite_note-:0-13">[9]</a> The hierarchical softmax (introduced by Morin and <a href="Yoshua_Bengio" title="Yoshua Bengio">Bengio</a> in 2005) uses a binary tree structure where the outcomes (vocabulary words) are the leaves and the intermediate nodes are suitably selected "classes" of outcomes, forming <a href="Latent_variable" class="mw-redirect" title="Latent variable">latent variables</a>.<a href="#cite_note-:1-14">[10]</a><a href="#cite_note-:2-15">[11]</a> The desired probability (softmax value) of a leaf (outcome) can then be calculated as the product of the probabilities of all nodes on the path from the root to that leaf.<a href="#cite_note-:1-14">[10]</a> Ideally, when the tree is balanced, this would reduce the <a href="Computational_complexity" title="Computational complexity">computational complexity</a> from <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle O(K)}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle O(K)}</annotation>

</semantics>

</math><img src="./21d6095b24973251be0d4d45dc1699053877217a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:5.649ex; height:2.843ex;" alt="{\displaystyle O(K)}" loading="lazy"> to <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle O(\log _{2}K)}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle O(\log _{2}K)}</annotation>

</semantics>

</math><img src="./6984560326928a3bfc59dc27018fb50ab706a7d9.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:10.062ex; height:2.843ex;" alt="{\displaystyle O(\log _{2}K)}" loading="lazy">.<a href="#cite_note-:2-15">[11]</a> In practice, results depend on choosing a good strategy for clustering the outcomes into classes.<a href="#cite_note-:1-14">[10]</a><a href="#cite_note-:2-15">[11]</a> A <a href="Huffman_tree" class="mw-redirect" title="Huffman tree">Huffman tree</a> was used for this in Google's <a href="Word2vec" title="Word2vec">word2vec</a> models (introduced in 2013) to achieve scalability.<a href="#cite_note-:0-13">[9]</a>

A second kind of remedies is based on approximating the softmax (during training) with modified loss functions that avoid the calculation of the full normalization factor.<a href="#cite_note-:0-13">[9]</a> These include methods that restrict the normalization sum to a sample of outcomes (e.g. Importance Sampling, Target Sampling).<a href="#cite_note-:0-13">[9]</a><a href="#cite_note-:1-14">[10]</a>

<div class="mw-heading mw-heading2"><h2 id="Numerical_algorithms">Numerical algorithms</h2></div>

The standard softmax is numerically unstable because of large exponentiations. The safe softmax method calculates instead<math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sigma (\mathbf {z} )_{i}={\frac {e^{\beta (z_{i}-m)}}{\sum _{j=1}^{K}e^{\beta (z_{j}-m)}}}}">

</mrow>

<msub>

</mrow>

</msub>

<mfrac>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

<mrow>

</mrow>

</mrow>

</munderover>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sigma (\mathbf {z} )_{i}={\frac {e^{\beta (z_{i}-m)}}{\sum _{j=1}^{K}e^{\beta (z_{j}-m)}}}}</annotation>

</semantics>

</math>where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle m=\max _{i}z_{i}}">

</mrow>

</munder>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle m=\max _{i}z_{i}}</annotation>

</semantics>

</math><img src="./64ec0f8fb2a352608bb80c6734ad1a40369eec83.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -2.005ex; width:11.733ex; height:3.343ex;" alt="{\displaystyle m=\max _{i}z_{i}}" loading="lazy"> is the largest factor involved. Subtracting by it guarantees that the exponentiations result in at most 1.

The <a href="Attention_(machine_learning)" title="Attention (machine learning)">attention mechanism</a> in <a href="Transformer_(deep_learning_architecture)" title="Transformer (deep learning architecture)">Transformers</a> takes three arguments: a "query vector" <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle q}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle q}</annotation>

</semantics>

</math><img src="./06809d64fa7c817ffc7e323f85997f783dbdf71d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:1.07ex; height:2.009ex;" alt="{\displaystyle q}" loading="lazy">, a list of "key vectors" <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle k_{1},\dots ,k_{N}}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle k_{1},\dots ,k_{N}}</annotation>

</semantics>

</math><img src="./3a8dae0dfa9068014a184a18a459a4787854cf36.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:10.346ex; height:2.509ex;" alt="{\displaystyle k_{1},\dots ,k_{N}}" loading="lazy">, and a list of "value vectors" <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle v_{1},\dots ,v_{N}}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle v_{1},\dots ,v_{N}}</annotation>

</semantics>

</math><img src="./b813c7562afc55a050f3306f3576e8728d3c55b6.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:10.179ex; height:2.009ex;" alt="{\displaystyle v_{1},\dots ,v_{N}}" loading="lazy">, and outputs a softmax-weighted sum over value vectors:<math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle o=\sum _{i=1}^{N}{\frac {e^{q^{T}k_{i}-m}}{\sum _{j=1}^{N}e^{q^{T}k_{j}-m}}}v_{i}}">

</mrow>

</mrow>

</munderover>

<mfrac>

<msup>

<msup>

</mrow>

</msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

<mrow>

</mrow>

</mrow>

</munderover>

<msup>

<msup>

</mrow>

</msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle o=\sum _{i=1}^{N}{\frac {e^{q^{T}k_{i}-m}}{\sum _{j=1}^{N}e^{q^{T}k_{j}-m}}}v_{i}}</annotation>

</semantics>

</math>The standard softmax method involves several loops over the inputs, which would be <a href="Memory_bottleneck" class="mw-redirect" title="Memory bottleneck">bottlenecked by memory bandwidth</a>. The <a href="FlashAttention" class="mw-redirect" title="FlashAttention">FlashAttention</a> method is a <a href="Communication-avoiding_algorithm" title="Communication-avoiding algorithm">communication-avoiding algorithm</a> that fuses these operations into a single loop, increasing the <a href="Arithmetic_intensity" class="mw-redirect" title="Arithmetic intensity">arithmetic intensity</a>. It is an <a href="Online_algorithm" title="Online algorithm">online algorithm</a> that computes the following quantities:<a href="#cite_note-16">[12]</a><a href="#cite_note-17">[13]</a><math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\begin{aligned}z_{i}&=q^{T}k_{i}&\\m_{i}&=\max(z_{1},\dots ,z_{i})&=&\max(m_{i-1},z_{i})\\l_{i}&=e^{z_{1}-m_{i}}+\dots +e^{z_{i}-m_{i}}&=&e^{m_{i-1}-m_{i}}l_{i-1}+e^{z_{i}-m_{i}}\\o_{i}&=e^{z_{1}-m_{i}}v_{1}+\dots +e^{z_{i}-m_{i}}v_{i}&=&e^{m_{i-1}-m_{i}}o_{i-1}+e^{z_{i}-m_{i}}v_{i}\end{aligned}}}">

<mtr>

<mtd>

<msub>

</mrow>

</msub>

</mtd>

<mtd>

<msup>

</mrow>

</msup>

<msub>

</mrow>

</msub>

</mtd>

</mtr>

<mtr>

<mtd>

<msub>

</mrow>

</msub>

</mtd>

<mtd>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mtd>

<mtd>

</mtd>

<mtd>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mtd>

</mtr>

<mtr>

<mtd>

<msub>

</mrow>

</msub>

</mtd>

<mtd>

<msup>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</msup>

<msup>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</msup>

</mtd>

<mtd>

</mtd>

<mtd>

<msup>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</msup>

<msub>

</mrow>

</msub>

<msup>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</msup>

</mtd>

</mtr>

<mtr>

<mtd>

<msub>

</mrow>

</msub>

</mtd>

<mtd>

<msup>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</msup>

<msub>

</mrow>

</msub>

<msup>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</msup>

<msub>

</mrow>

</msub>

</mtd>

<mtd>

</mtd>

<mtd>

<msup>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</msup>

<msub>

</mrow>

</msub>

<msup>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</msup>

<msub>

</mrow>

</msub>

</mtd>

</mtr>

</mtable>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\begin{aligned}z_{i}&=q^{T}k_{i}&\\m_{i}&=\max(z_{1},\dots ,z_{i})&=&\max(m_{i-1},z_{i})\\l_{i}&=e^{z_{1}-m_{i}}+\dots +e^{z_{i}-m_{i}}&=&e^{m_{i-1}-m_{i}}l_{i-1}+e^{z_{i}-m_{i}}\\o_{i}&=e^{z_{1}-m_{i}}v_{1}+\dots +e^{z_{i}-m_{i}}v_{i}&=&e^{m_{i-1}-m_{i}}o_{i-1}+e^{z_{i}-m_{i}}v_{i}\end{aligned}}}</annotation>

</semantics>

</math>and returns <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle o_{N}/l_{N}}">

<msub>

</mrow>

</msub>

</mrow>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle o_{N}/l_{N}}</annotation>

</semantics>

</math><img src="./9a4d67e3df09073613aa04e047c272961b8fe3bc.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:6.366ex; height:2.843ex;" alt="{\displaystyle o_{N}/l_{N}}" loading="lazy">. In practice, FlashAttention operates over multiple queries and keys per loop iteration, in a similar way as <a href="Communication-avoiding_algorithm#Blocked_(tiled)_matrix_multiplication" title="Communication-avoiding algorithm">blocked matrix multiplication</a>. If <a href="Backpropagation" title="Backpropagation">backpropagation</a> is needed, then the output vectors and the intermediate arrays <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle [m_{1},\dots ,m_{N}],[l_{1},\dots ,l_{N}]}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle [m_{1},\dots ,m_{N}],[l_{1},\dots ,l_{N}]}</annotation>

</semantics>

</math><img src="./59e9527dea2ec3c28a24be76f328c7ac46a0706f.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:24.936ex; height:2.843ex;" alt="{\displaystyle [m_{1},\dots ,m_{N}],[l_{1},\dots ,l_{N}]}" loading="lazy"> are cached, and during the backward pass, attention matrices are <a href="Rematerialization" title="Rematerialization">rematerialized</a> from these, making it a form of gradient checkpointing.

<div class="mw-heading mw-heading2"><h2 id="Mathematical_properties">Mathematical properties</h2></div>

Geometrically the softmax function maps the <a href="Euclidean_space" title="Euclidean space">Euclidean space</a> <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathbb {R} ^{K}}">

<msup>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mathbb {R} ^{K}}</annotation>

</semantics>

</math><img src="./a7fe4c8791d7d313b6c2f1ddca68b6662fcaf60a.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:3.371ex; height:2.676ex;" alt="{\displaystyle \mathbb {R} ^{K}}" loading="lazy"> to the <a href="Boundary_(topology)" title="Boundary (topology)">boundary</a> of the <a href="Standard_simplex" class="mw-redirect" title="Standard simplex">standard <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (K-1)}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle (K-1)}</annotation>

</semantics>

</math><img src="./165942a48ca8aa12b7ef27c0b10e3d1a358df663.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:7.878ex; height:2.843ex;" alt="{\displaystyle (K-1)}" loading="lazy">-simplex</a>, cutting the dimension by one (the range is a <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (K-1)}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle (K-1)}</annotation>

</semantics>

</math><img src="./165942a48ca8aa12b7ef27c0b10e3d1a358df663.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:7.878ex; height:2.843ex;" alt="{\displaystyle (K-1)}" loading="lazy">-dimensional simplex in <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle K}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle K}</annotation>

</semantics>

</math><img src="./2b76fce82a62ed5461908f0dc8f037de4e3686b0.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.066ex; height:2.176ex;" alt="{\displaystyle K}" loading="lazy">-dimensional space), due to the <a href="Linear_constraint" class="mw-redirect" title="Linear constraint">linear constraint</a> that all output sum to 1 meaning it lies on a <a href="Hyperplane" title="Hyperplane">hyperplane</a>.

Along the main diagonal <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (x,\,x,\,\dots ,\,x),}">

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle (x,\,x,\,\dots ,\,x),}</annotation>

</semantics>

</math><img src="./2760f7edf05f6d6c0f6d576eb09f5c2b860a335f.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:13.819ex; height:2.843ex;" alt="{\displaystyle (x,\,x,\,\dots ,\,x),}" loading="lazy"> softmax is just the uniform distribution on outputs, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (1/n,\dots ,1/n)}">

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle (1/n,\dots ,1/n)}</annotation>

</semantics>

</math><img src="./e9c5153fb398b8ae6def33c6f28997bc9f1128c9.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:14.427ex; height:2.843ex;" alt="{\displaystyle (1/n,\dots ,1/n)}" loading="lazy">: equal scores yield equal probabilities.

More generally, softmax is invariant under translation by the same value in each coordinate: adding <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathbf {c} =(c,\,\dots ,\,c)}">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mathbf {c} =(c,\,\dots ,\,c)}</annotation>

</semantics>

</math><img src="./f075eafd6ea39439a774f527110f90b4317d934b.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:14.062ex; height:2.843ex;" alt="{\displaystyle \mathbf {c} =(c,\,\dots ,\,c)}" loading="lazy"> to the inputs <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathbf {z} }">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mathbf {z} }</annotation>

</semantics>

</math><img src="./82eca5d0928078d5a61b9e7e98cc73db31070909.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.188ex; height:1.676ex;" alt="{\displaystyle \mathbf {z} }" loading="lazy"> yields <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sigma (\mathbf {z} +\mathbf {c} )=\sigma (\mathbf {z} )}">

</mrow>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sigma (\mathbf {z} +\mathbf {c} )=\sigma (\mathbf {z} )}</annotation>

</semantics>

</math><img src="./8b37a2cecd44ed20eae0c502ea6d2ccbf0eee7e5.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:15.781ex; height:2.843ex;" alt="{\displaystyle \sigma (\mathbf {z} +\mathbf {c} )=\sigma (\mathbf {z} )}" loading="lazy">, because it multiplies each exponent by the same factor, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle e^{c}}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle e^{c}}</annotation>

</semantics>

</math><img src="./c1bf25eaed686bee0625d513045b7150a6d0d684.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:2.028ex; height:2.343ex;" alt="{\displaystyle e^{c}}" loading="lazy"> (because <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle e^{z_{i}+c}=e^{z_{i}}\cdot e^{c}}">

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle e^{z_{i}+c}=e^{z_{i}}\cdot e^{c}}</annotation>

</semantics>

</math><img src="./37845967b81e0161119d677bc37454d079e9b2f3.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:14.206ex; height:2.509ex;" alt="{\displaystyle e^{z_{i}+c}=e^{z_{i}}\cdot e^{c}}" loading="lazy">), so the ratios do not change:

</mrow>

</mrow>

<msub>

</mrow>

</msub>

<mfrac>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

<mrow>

</mrow>

</mrow>

</munderover>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

<mfrac>

<mrow>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

<msup>

</mrow>

</msup>

</mrow>

<mrow>

</mrow>

</mrow>

</munderover>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

<msup>

</mrow>

</msup>

</mrow>

</mfrac>

</mrow>

</mrow>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sigma (\mathbf {z} +\mathbf {c} )_{j}={\frac {e^{z_{j}+c}}{\sum _{k=1}^{K}e^{z_{k}+c}}}={\frac {e^{z_{j}}\cdot e^{c}}{\sum _{k=1}^{K}e^{z_{k}}\cdot e^{c}}}=\sigma (\mathbf {z} )_{j}.}</annotation>

</semantics>

</math>

Geometrically, softmax is constant along diagonals: this is the dimension that is eliminated, and corresponds to the softmax output being independent of a translation in the input scores (a choice of 0 score). One can normalize input scores by assuming that the sum is zero (subtract the average: <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \mathbf {c} }">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \mathbf {c} }</annotation>

</semantics>

</math><img src="./8798d172f59e21f2ce193a3118d4063d19353ded.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:1.188ex; height:1.676ex;" alt="{\displaystyle \mathbf {c} }" loading="lazy"> where <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\textstyle c={\frac {1}{n}}\sum z_{i}}">

<mfrac>

</mfrac>

</mrow>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\textstyle c={\frac {1}{n}}\sum z_{i}}</annotation>

</semantics>

</math><img src="./a0890a9fb83b6fe2f2b14c63b6e43ceb90ea2b94.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:11.037ex; height:3.343ex;" alt="{\textstyle c={\frac {1}{n}}\sum z_{i}}" loading="lazy">), and then the softmax takes the hyperplane of points that sum to zero, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\textstyle \sum z_{i}=0}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\textstyle \sum z_{i}=0}</annotation>

</semantics>

</math><img src="./f03018321383784c779ea4449d6e4cbd0059a0cf.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:8.983ex; height:2.843ex;" alt="{\textstyle \sum z_{i}=0}" loading="lazy">, to the open simplex of positive values that sum to 1<math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\textstyle \sum \sigma (\mathbf {z} )_{i}=1}">

</mrow>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\textstyle \sum \sigma (\mathbf {z} )_{i}=1}</annotation>

</semantics>

</math><img src="./f55e4d5660c3fe5c98724d85e07c1f8fd7c34258.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:12.228ex; height:2.843ex;" alt="{\textstyle \sum \sigma (\mathbf {z} )_{i}=1}" loading="lazy">, analogously to how the exponent takes 0 to 1, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle e^{0}=1}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle e^{0}=1}</annotation>

</semantics>

</math><img src="./ea9335287ddbe5eb6cf606a44f58d2acf782a979.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:6.399ex; height:2.676ex;" alt="{\displaystyle e^{0}=1}" loading="lazy"> and is positive.

By contrast, softmax is not invariant under scaling. For instance, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sigma {\bigl (}(0,\,1){\bigr )}={\bigl (}1/(1+e),\,e/(1+e){\bigr )}}">

</mrow>

</mrow>

</mrow>

</mrow>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sigma {\bigl (}(0,\,1){\bigr )}={\bigl (}1/(1+e),\,e/(1+e){\bigr )}}</annotation>

</semantics>

</math><img src="./553e1aea647bf31712f2eb386bdb3a4274cbab37.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:34.026ex; height:3.176ex;" alt="{\displaystyle \sigma {\bigl (}(0,\,1){\bigr )}={\bigl (}1/(1+e),\,e/(1+e){\bigr )}}" loading="lazy"> but <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \sigma {\bigl (}(0,2){\bigr )}={\bigl (}1/\left(1+e^{2}\right),\,e^{2}/\left(1+e^{2}\right){\bigr )}.}">

</mrow>

</mrow>

</mrow>

</mrow>

<mrow>

<mrow>

<msup>

</mrow>

</msup>

</mrow>

</mrow>

<msup>

</mrow>

</msup>

</mrow>

<mrow>

<mrow>

<msup>

</mrow>

</msup>

</mrow>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \sigma {\bigl (}(0,2){\bigr )}={\bigl (}1/\left(1+e^{2}\right),\,e^{2}/\left(1+e^{2}\right){\bigr )}.}</annotation>

</semantics>

</math><img src="./fafe1fb7275fdf238aeb5e605531adf7d7ece0d5.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:39.638ex; height:3.343ex;" alt="{\displaystyle \sigma {\bigl (}(0,2){\bigr )}={\bigl (}1/\left(1+e^{2}\right),\,e^{2}/\left(1+e^{2}\right){\bigr )}.}" loading="lazy">

The <a href="Standard_logistic_function" class="mw-redirect" title="Standard logistic function">standard logistic function</a> is the special case for a 1-dimensional axis in 2-dimensional space, say the x-axis in the (x, y) plane. One variable is fixed at 0 (say <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle z_{2}=0}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle z_{2}=0}</annotation>

</semantics>

</math><img src="./23b27d6d9c1abc0b7f47701d1fca0d8f5a29485d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:6.396ex; height:2.509ex;" alt="{\displaystyle z_{2}=0}" loading="lazy">), so <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle e^{0}=1}">

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle e^{0}=1}</annotation>

</semantics>

</math><img src="./ea9335287ddbe5eb6cf606a44f58d2acf782a979.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:6.399ex; height:2.676ex;" alt="{\displaystyle e^{0}=1}" loading="lazy">, and the other variable can vary, denote it <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle z_{1}=x}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle z_{1}=x}</annotation>

</semantics>

</math><img src="./35c46cdddef441e805338a2e9ac5961d1aaaaab9.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:6.564ex; height:2.009ex;" alt="{\displaystyle z_{1}=x}" loading="lazy">, so <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\textstyle e^{z_{1}}/\sum _{k=1}^{2}e^{z_{k}}=e^{x}/\left(e^{x}+1\right),}">

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mrow>

</mrow>

</munderover>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

<msup>

</mrow>

</msup>

</mrow>

<mrow>

<mrow>

<msup>

</mrow>

</msup>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\textstyle e^{z_{1}}/\sum _{k=1}^{2}e^{z_{k}}=e^{x}/\left(e^{x}+1\right),}</annotation>

</semantics>

</math><img src="./ed1d7bdb419bdf14b2a8e86da178102e88db21f0.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:29.437ex; height:3.509ex;" alt="{\textstyle e^{z_{1}}/\sum _{k=1}^{2}e^{z_{k}}=e^{x}/\left(e^{x}+1\right),}" loading="lazy"> the standard logistic function, and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\textstyle e^{z_{2}}/\sum _{k=1}^{2}e^{z_{k}}=1/\left(e^{x}+1\right),}">

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mrow>

</mrow>

</munderover>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

<mrow>

<mrow>

<msup>

</mrow>

</msup>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\textstyle e^{z_{2}}/\sum _{k=1}^{2}e^{z_{k}}=1/\left(e^{x}+1\right),}</annotation>

</semantics>

</math><img src="./d843ef1f6b6e769e69de07c0e895dbab2fedd180.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:28.344ex; height:3.509ex;" alt="{\textstyle e^{z_{2}}/\sum _{k=1}^{2}e^{z_{k}}=1/\left(e^{x}+1\right),}" loading="lazy"> its complement (meaning they add up to 1). The 1-dimensional input could alternatively be expressed as the line <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (x/2,\,-x/2)}">

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle (x/2,\,-x/2)}</annotation>

</semantics>

</math><img src="./cd9e620ff83e8eb86183c87ab39025fda6e861d7.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:12.348ex; height:2.843ex;" alt="{\displaystyle (x/2,\,-x/2)}" loading="lazy">, with outputs <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle e^{x/2}/\left(e^{x/2}+e^{-x/2}\right)=e^{x}/\left(e^{x}+1\right)}">

<msup>

</mrow>

</mrow>

</msup>

</mrow>

<mrow>

<mrow>

<msup>

</mrow>

</mrow>

</msup>

<msup>

</mrow>

</mrow>

</msup>

</mrow>

</mrow>

<msup>

</mrow>

</msup>

</mrow>

<mrow>

<mrow>

<msup>

</mrow>

</msup>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle e^{x/2}/\left(e^{x/2}+e^{-x/2}\right)=e^{x}/\left(e^{x}+1\right)}</annotation>

</semantics>

</math><img src="./679a08cc268a46b53c4b28e54a95287fad8ca6f7.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.838ex; width:35.116ex; height:4.843ex;" alt="{\displaystyle e^{x/2}/\left(e^{x/2}+e^{-x/2}\right)=e^{x}/\left(e^{x}+1\right)}" loading="lazy"> and <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle e^{-x/2}/\left(e^{x/2}+e^{-x/2}\right)=1/\left(e^{x}+1\right).}">

<msup>

</mrow>

</mrow>

</msup>

</mrow>

<mrow>

<mrow>

<msup>

</mrow>

</mrow>

</msup>

<msup>

</mrow>

</mrow>

</msup>

</mrow>

</mrow>

</mrow>

<mrow>

<mrow>

<msup>

</mrow>

</msup>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle e^{-x/2}/\left(e^{x/2}+e^{-x/2}\right)=1/\left(e^{x}+1\right).}</annotation>

</semantics>

</math><img src="./430af75c2d433a51e520961a758aa34b0d18df04.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.838ex; width:36.335ex; height:4.843ex;" alt="{\displaystyle e^{-x/2}/\left(e^{x/2}+e^{-x/2}\right)=1/\left(e^{x}+1\right).}" loading="lazy">

<div class="mw-heading mw-heading3"><h3 id="Gradients">Gradients</h3></div>

The softmax function is also the gradient of the <a href="LogSumExp" title="LogSumExp">LogSumExp</a> function:<math display="block" xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle {\frac {\partial }{\partial z_{i}}}\operatorname {LSE} (\mathbf {z} )={\frac {\exp z_{i}}{\sum _{j=1}^{K}\exp z_{j}}}=\sigma (\mathbf {z} )_{i},\quad {\text{ for }}i=1,\dotsc ,K,\quad \mathbf {z} =(z_{1},\,\dotsc ,\,z_{K})\in \mathbb {R} ^{K},}">

<mfrac>

<mrow>

<msub>

</mrow>

</msub>

</mrow>

</mfrac>

</mrow>

</mrow>

<mfrac>

<mrow>

<msub>

</mrow>

</msub>

</mrow>

<mrow>

</mrow>

</mrow>

</munderover>

<msub>

</mrow>

</msub>

</mrow>

</mfrac>

</mrow>

</mrow>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msup>

</mrow>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle {\frac {\partial }{\partial z_{i}}}\operatorname {LSE} (\mathbf {z} )={\frac {\exp z_{i}}{\sum _{j=1}^{K}\exp z_{j}}}=\sigma (\mathbf {z} )_{i},\quad {\text{ for }}i=1,\dotsc ,K,\quad \mathbf {z} =(z_{1},\,\dotsc ,\,z_{K})\in \mathbb {R} ^{K},}</annotation>

</semantics>

</math>where the LogSumExp function is defined as <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {LSE} (z_{1},\,\dots ,\,z_{n})=\log \left(\exp(z_{1})+\cdots +\exp(z_{n})\right)}">

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<mrow>

<mrow>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \operatorname {LSE} (z_{1},\,\dots ,\,z_{n})=\log \left(\exp(z_{1})+\cdots +\exp(z_{n})\right)}</annotation>

</semantics>

</math><img src="./ef7322c915a50736e96c6bfe978f1d7b8f53536d.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:47.967ex; height:2.843ex;" alt="{\displaystyle \operatorname {LSE} (z_{1},\,\dots ,\,z_{n})=\log \left(\exp(z_{1})+\cdots +\exp(z_{n})\right)}" loading="lazy">.

The gradient of softmax is thus <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \partial _{z_{j}}\sigma _{i}=\sigma _{i}(\delta _{ij}-\sigma _{j})}">

<msub>

<msub>

</mrow>

</msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \partial _{z_{j}}\sigma _{i}=\sigma _{i}(\delta _{ij}-\sigma _{j})}</annotation>

</semantics>

</math><img src="./23bf5dcf501a4ce13916c26bde085dfab06f8392.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.171ex; width:19.694ex; height:3.176ex;" alt="{\displaystyle \partial _{z_{j}}\sigma _{i}=\sigma _{i}(\delta _{ij}-\sigma _{j})}" loading="lazy">.

<div class="mw-heading mw-heading2"><h2 id="History">History</h2></div>

The softmax function was used in <a href="Statistical_mechanics" title="Statistical mechanics">statistical mechanics</a> as the <a href="Boltzmann_distribution" title="Boltzmann distribution">Boltzmann distribution</a> in the foundational paper <a href="#CITEREFBoltzmann1868">Boltzmann (1868)</a>,<a href="#cite_note-18">[14]</a> formalized and popularized in the influential textbook <a href="#CITEREFGibbs1902">Gibbs (1902)</a>.<a href="#cite_note-19">[15]</a>

The use of the softmax in <a href="Decision_theory" title="Decision theory">decision theory</a> is credited to <a href="R._Duncan_Luce" title="R. Duncan Luce">R. Duncan Luce</a>,<a href="#cite_note-Gao-20">[16]</a>: 1 who used the axiom of <a href="Independence_of_irrelevant_alternatives" title="Independence of irrelevant alternatives">independence of irrelevant alternatives</a> in <a href="Rational_choice_theory" class="mw-redirect" title="Rational choice theory">rational choice theory</a> to deduce the softmax in <a href="Luce's_choice_axiom" title="Luce's choice axiom">Luce's choice axiom</a> for relative preferences.

In machine learning, the term "softmax" is credited to John S. Bridle in two 1989 conference papers, <a href="#CITEREFBridle1990a">Bridle (1990a)</a>:<a href="#cite_note-Gao-20">[16]</a>: 1 and <a href="#CITEREFBridle1990b">Bridle (1990b)</a>:<a href="#cite_note-sako2018-7">[3]</a>

/* start https://en.wikipedia.org/ */

.mw-parser-output .templatequote{overflow:hidden;margin:1em 0;padding:0 32px}.mw-parser-output .templatequotecite{line-height:1.5em;text-align:left;margin-top:0}@media(min-width:500px){.mw-parser-output .templatequotecite{padding-left:1.6em}}

/* end https://en.wikipedia.org/ */

</style><blockquote class="templatequote">We are concerned with feed-forward non-linear networks (multi-layer perceptrons, or MLPs) with multiple outputs. We wish to treat the outputs of the network as probabilities of alternatives (e.g. pattern classes), conditioned on the inputs. We look for appropriate output non-linearities and for appropriate criteria for adaptation of the parameters of the network (e.g. weights). We explain two modifications: probability scoring, which is an alternative to squared error minimisation, and a normalised exponential (softmax) multi-input generalisation of the logistic non-linearity.<a href="#cite_note-21">[17]</a>: 227

</blockquote>

<blockquote class="templatequote">For any input, the outputs must all be positive and they must sum to unity. ...

Given a set of unconstrained values, ⁠<math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle V_{j}(x)}">

<msub>

</mrow>

</msub>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle V_{j}(x)}</annotation>

</semantics>

</math><img src="./ba18246a022eca009ca18b825e1c2802b44c47f4.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -1.005ex; width:5.404ex; height:3.009ex;" alt="{\displaystyle V_{j}(x)}" loading="lazy">⁠, we can ensure both conditions by using a Normalised Exponential transformation:

<msub>

</mrow>

</msub>

<mrow>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

</mrow>

</mrow>

</munder>

<msup>

<msub>

</mrow>

</msub>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle Q_{j}(x)=\left.e^{V_{j}(x)}\right/\sum _{k}e^{V_{k}(x)}}</annotation>

</semantics>

</math>

This transformation can be considered a multi-input generalisation of the logistic, operating on the whole output layer. It preserves the rank order of its input values, and is a differentiable generalisation of the 'winner-take-all' operation of picking the maximum value. For this reason we like to refer to it as softmax.<a href="#cite_note-22">[18]</a>: 213

</blockquote>

<div class="mw-heading mw-heading2"><h2 id="Example">Example</h2></div>

With an input of (1, 2, 3, 4, 1, 2, 3), the softmax is approximately (0.024, 0.064, 0.175, 0.475, 0.024, 0.064, 0.175). The output has most of its weight where the "4" was in the original input. This is what the function is normally used for: to highlight the largest values and suppress values which are significantly below the maximum value. But note: a change of temperature changes the output. When the temperature is multiplied by 10, the inputs are effectively (0.1, 0.2, 0.3, 0.4, 0.1, 0.2, 0.3) and the softmax is approximately (0.125, 0.138, 0.153, 0.169, 0.125, 0.138, 0.153). This shows that high temperatures de-emphasize the maximum value.

Computation of this example using <a href="Python_(programming_language)" title="Python (programming language)">Python</a> code:

<div class="mw-highlight mw-highlight-lang-pycon mw-content-ltr" dir="ltr"><pre>>>> import numpy as np

>>> z = np.array([1.0, 2.0, 3.0, 4.0, 1.0, 2.0, 3.0])

>>> beta = 1.0

>>> np.exp(beta * z) / np.sum(np.exp(beta * z))

array([0.02364054, 0.06426166, 0.1746813, 0.474833, 0.02364054,

0.06426166, 0.1746813])

</pre></div>

<div class="mw-heading mw-heading2"><h2 id="Alternatives">Alternatives</h2></div>

The softmax function generates probability predictions densely distributed over its <a href="Support_(mathematics)" title="Support (mathematics)">support</a>. Other functions like sparsemax or α-entmax can be used when sparse probability predictions are desired.<a href="#cite_note-23">[19]</a> Also the <a href="Reparametrization_trick" class="mw-redirect" title="Reparametrization trick">Gumbel-softmax reparametrization trick</a> can be used when sampling from a discrete-discrete distribution needs to be mimicked in a differentiable manner.

<ul><li><a href="Softplus" title="Softplus">Softplus</a></li>

<li><a href="Multinomial_logistic_regression" title="Multinomial logistic regression">Multinomial logistic regression</a></li>

<li><a href="Dirichlet_distribution" title="Dirichlet distribution">Dirichlet distribution</a> – an alternative way to sample categorical distributions</li>

<li><a href="Partition_function_(statistical_mechanics)" title="Partition function (statistical mechanics)">Partition function</a></li>

<li><a href="Exponential_tilting" title="Exponential tilting">Exponential tilting</a> – a generalization of Softmax to more general probability distributions</li></ul>

<div class="mw-heading mw-heading2"><h2 id="Notes">Notes</h2></div>

/* start https://en.wikipedia.org/ */

.mw-parser-output .reflist{margin-bottom:0.5em;list-style-type:decimal}@media screen{.mw-parser-output .reflist{font-size:90%}}.mw-parser-output .reflist .references{font-size:100%;margin-bottom:0;list-style-type:inherit}.mw-parser-output .reflist-columns-2{column-width:30em}.mw-parser-output .reflist-columns-3{column-width:25em}.mw-parser-output .reflist-columns{margin-top:0.3em}.mw-parser-output .reflist-columns ol{margin-top:0}.mw-parser-output .reflist-columns li{page-break-inside:avoid;break-inside:avoid-column}.mw-parser-output .reflist-upper-alpha{list-style-type:upper-alpha}.mw-parser-output .reflist-upper-roman{list-style-type:upper-roman}.mw-parser-output .reflist-lower-alpha{list-style-type:lower-alpha}.mw-parser-output .reflist-lower-greek{list-style-type:lower-greek}.mw-parser-output .reflist-lower-roman{list-style-type:lower-roman}

/* end https://en.wikipedia.org/ */

</style><div class="reflist reflist-lower-alpha">

<li id="cite_note-3"><a href="#cite_ref-3">^</a> Positive β corresponds to the maximum convention, and is usual in machine learning, corresponding to the highest score having highest probability. The negative −β corresponds to the minimum convention, and is conventional in thermodynamics, corresponding to the lowest energy state having the highest probability; this matches the convention in the <a href="Gibbs_distribution" class="mw-redirect" title="Gibbs distribution">Gibbs distribution</a>, interpreting β as <a href="Coldness" class="mw-redirect" title="Coldness">coldness</a>.

</li>

<li id="cite_note-4"><a href="#cite_ref-4">^</a> The notation β is for the <a href="Thermodynamic_beta" title="Thermodynamic beta">thermodynamic beta</a>, which is inverse <a href="Temperature" title="Temperature">temperature</a>: <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \beta =1/T}">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \beta =1/T}</annotation>

</semantics>

</math><img src="./cf1e21c8b3cc0966416fbb167dc0f257e83fd65e.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:8.392ex; height:2.843ex;" alt="{\displaystyle \beta =1/T}" loading="lazy">, <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle T=1/\beta .}">

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle T=1/\beta .}</annotation>

</semantics>

</math><img src="./46ecb95c1a7c6b05514974d9cc18934cf8a3ceba.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:9.038ex; height:2.843ex;" alt="{\displaystyle T=1/\beta .}" loading="lazy">

</li>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \beta =0}</annotation>

</semantics>

</math><img src="./60b5e78663eba7ba08e0dd4915251e6261f4f35c.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.671ex; width:5.593ex; height:2.509ex;" alt="{\displaystyle \beta =0}" loading="lazy"> (<a href="Coldness" class="mw-redirect" title="Coldness">coldness</a> zero, infinite temperature), <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle b=e^{\beta }=e^{0}=1}">

<msup>

</mrow>

</msup>

<msup>

</mrow>

</msup>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle b=e^{\beta }=e^{0}=1}</annotation>

</semantics>

</math><img src="./44fe48a44d1efa08ae3b603e6500dde601bf0747.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.338ex; width:15.851ex; height:2.676ex;" alt="{\displaystyle b=e^{\beta }=e^{0}=1}" loading="lazy">, and this becomes the constant function ⁠<math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle (1/n,\dots ,1/n)}">

</mrow>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle (1/n,\dots ,1/n)}</annotation>

</semantics>

</math><img src="./e9c5153fb398b8ae6def33c6f28997bc9f1128c9.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:14.427ex; height:2.843ex;" alt="{\displaystyle (1/n,\dots ,1/n)}" loading="lazy">⁠, corresponding to the <a href="Discrete_uniform_distribution" title="Discrete uniform distribution">discrete uniform distribution</a>.

</li>

<li id="cite_note-6"><a href="#cite_ref-6">^</a> In statistical mechanics, fixing β is interpreted as having coldness and temperature of 1.

</li>

</ol></div></div>

<div class="mw-heading mw-heading2"><h2 id="References">References</h2></div>

/* start https://en.wikipedia.org/ */

.mw-parser-output cite.citation{font-style:inherit;word-wrap:break-word}.mw-parser-output .citation q{quotes:"\"""\"""'""'"}.mw-parser-output .citation:target{background-color:rgba(0,127,255,0.133)}.mw-parser-output .id-lock-free.id-lock-free a{background:url("./mw/Lock-green.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-limited.id-lock-limited a,.mw-parser-output .id-lock-registration.id-lock-registration a{background:url("./mw/Lock-gray-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .id-lock-subscription.id-lock-subscription a{background:url("./mw/Lock-red-alt-2.svg")right 0.1em center/9px no-repeat}.mw-parser-output .cs1-ws-icon a{background:url("./mw/Wikisource-logo.svg")right 0.1em center/12px no-repeat}body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-free a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-limited a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-registration a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .id-lock-subscription a,body:not(.skin-timeless):not(.skin-minerva) .mw-parser-output .cs1-ws-icon a{background-size:contain;padding:0 1em 0 0}.mw-parser-output .cs1-code{color:inherit;background:inherit;border:none;padding:inherit}.mw-parser-output .cs1-hidden-error{display:none;color:var(--color-error,#d33)}.mw-parser-output .cs1-visible-error{color:var(--color-error,#d33)}.mw-parser-output .cs1-maint{display:none;color:#085;margin-left:0.3em}.mw-parser-output .cs1-kern-left{padding-left:0.2em}.mw-parser-output .cs1-kern-right{padding-right:0.2em}.mw-parser-output .citation .mw-selflink{font-weight:inherit}@media screen{.mw-parser-output .cs1-format{font-size:95%}html.skin-theme-clientpref-night .mw-parser-output .cs1-maint{color:#18911f}}@media screen and (prefers-color-scheme:dark){html.skin-theme-clientpref-os .mw-parser-output .cs1-maint{color:#18911f}}

/* end https://en.wikipedia.org/ */

</style><cite id="CITEREFGoodfellowBengioCourville2016" class="citation book cs1"><a href="Ian_Goodfellow" title="Ian Goodfellow">Goodfellow, Ian</a>; <a href="Yoshua_Bengio" title="Yoshua Bengio">Bengio, Yoshua</a>; Courville, Aaron (2016). <a rel="nofollow" class="external text" href="https://www.deeplearningbook.org/contents/mlp.html#pf15">"6.2.2.3 Softmax Units for Multinoulli Output Distributions"</a>. <a rel="nofollow" class="external text" href="http://www.deeplearningbook.org">Deep Learning</a>. MIT Press. pp. 180–184. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-0-26203561-3</bdi>.</cite>

</li>

<li id="cite_note-bishop-2">^ <a href="#cite_ref-bishop_2-0">a</a> <a href="#cite_ref-bishop_2-1">b</a> <cite id="CITEREFBishop2006" class="citation book cs1">Bishop, Christopher M. (2006). Pattern Recognition and Machine Learning. Springer. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>0-387-31073-8</bdi>.</cite>

</li>

<li id="cite_note-sako2018-7">^ <a href="#cite_ref-sako2018_7-0">a</a> <a href="#cite_ref-sako2018_7-1">b</a> <cite id="CITEREFSako2018" class="citation web cs1">Sako, Yusaku (2018-06-02). <a rel="nofollow" class="external text" href="https://medium.com/@u39kun/is-the-term-softmax-driving-you-nuts-ee232ab4f6bd">"Is the term "softmax" driving you nuts?"</a>. <a href="Medium_(website)" title="Medium (website)">Medium</a>.</cite>

</li>

<li id="cite_note-FOOTNOTEGoodfellowBengioCourville2016183–184-8"><a href="#cite_ref-FOOTNOTEGoodfellowBengioCourville2016183–184_8-0">^</a> <a href="#CITEREFGoodfellowBengioCourville2016">Goodfellow, Bengio & Courville 2016</a>, pp. 183–184: The name "softmax" can be somewhat confusing. The function is more closely related to the arg max function than the max function. The term "soft" derives from the fact that the softmax function is continuous and differentiable. The arg max function, with its result represented as a one-hot vector, is not continuous nor differentiable. The softmax function thus provides a "softened" version of the arg max. The corresponding soft version of the maximum function is <math xmlns="http://www.w3.org/1998/Math/MathML" alttext="{\displaystyle \operatorname {softmax} (\mathbf {z} )^{\top }\mathbf {z} }">

<mi>softmax</mi>

</mrow>

<msup>

</mrow>

</msup>

</mrow>

</mstyle>

</mrow>

<annotation encoding="application/x-tex">{\displaystyle \operatorname {softmax} (\mathbf {z} )^{\top }\mathbf {z} }</annotation>

</semantics>

</math><img src="./24df461e54e80e1eff826afc549917d0974f8ce4.svg" class="mwe-math-fallback-image-inline mw-invert skin-invert" aria-hidden="true" style="vertical-align: -0.838ex; width:13.717ex; height:3.176ex;" alt="{\displaystyle \operatorname {softmax} (\mathbf {z} )^{\top }\mathbf {z} }" loading="lazy">. It would perhaps be better to call the softmax function "softargmax," but the current name is an entrenched convention.

</li>

<li id="cite_note-9"><a href="#cite_ref-9">^</a> <cite id="CITEREFLeCunChopraHadsellRanzato2006" class="citation book cs1"><a href="Yann_LeCun" title="Yann LeCun">LeCun, Yann</a>; Chopra, Sumit; Hadsell, Raia; Ranzato, Marc’Aurelio; Huang, Fu Jie (2006). <a rel="nofollow" class="external text" href="http://yann.lecun.com/exdb/publis/pdf/lecun-06.pdf">"A Tutorial on Energy-Based Learning"</a> (PDF). In Gökhan Bakır; Thomas Hofmann; Bernhard Schölkopf; Alexander J. Smola; Ben Taskar; S.V.N Vishwanathan (eds.). <a rel="nofollow" class="external text" href="https://mitpress.mit.edu/books/predicting-structured-data">Predicting Structured Data</a>. Neural Information Processing series. MIT Press. <a href="ISBN_(identifier)" class="mw-redirect" title="ISBN (identifier)">ISBN</a> <bdi>978-0-26202617-8</bdi>.</cite>

</li>

<li id="cite_note-10"><a href="#cite_ref-10">^</a> <cite class="citation web cs1"><a rel="nofollow" class="external text" href="http://ufldl.stanford.edu/tutorial/supervised/SoftmaxRegression/">"Unsupervised Feature Learning and Deep Learning Tutorial"</a>. ufldl.stanford.edu. Retrieved 2024-03-25.</cite>

</li>

<li id="cite_note-11"><a href="#cite_ref-11">^</a> ai-faq <a rel="nofollow" class="external text" href="http://www.faqs.org/faqs/ai-faq/neural-nets/part2/section-12.html">What is a softmax activation function?</a>

</li>

<li id="cite_note-12"><a href="#cite_ref-12">^</a> Sutton, R. S. and Barto A. G. Reinforcement Learning: An Introduction. The MIT Press, Cambridge, MA, 1998. <a rel="nofollow" class="external text" href="http://incompleteideas.net/book/ebook/node17.html">Softmax Action Selection</a>

</li>

<li id="cite_note-:0-13">^ <a href="#cite_ref-:0_13-0">a</a> <a href="#cite_ref-:0_13-1">b</a> <a href="#cite_ref-:0_13-2">c</a> <a href="#cite_ref-:0_13-3">d</a> <a href="#cite_ref-:0_13-4">e</a> <a href="#cite_ref-:0_13-5">f</a> <a href="#cite_ref-:0_13-6">g</a> <cite id="CITEREFOnalZhangAltingovdeRahman2018" class="citation journal cs1">Onal, Kezban Dilek; Zhang, Ye; Altingovde, Ismail Sengor; Rahman, Md Mustafizur; Karagoz, Pinar; Braylan, Alex; Dang, Brandon; Chang, Heng-Lu; Kim, Henna; McNamara, Quinten; Angert, Aaron (2018-06-01). <a rel="nofollow" class="external text" href="https://doi.org/10.1007%2Fs10791-017-9321-y">"Neural information retrieval: at the end of the early years"</a>. Information Retrieval Journal. 21 (2): 111–182. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2Fs10791-017-9321-y">10.1007/s10791-017-9321-y</a>. <a href="Hdl_(identifier)" class="mw-redirect" title="Hdl (identifier)">hdl</a>:<a rel="nofollow" class="external text" href="https://hdl.handle.net/11245.1%2F008d6e8f-df13-4abf-8ae9-6ff2e17377f3">11245.1/008d6e8f-df13-4abf-8ae9-6ff2e17377f3</a>. <a href="ISSN_(identifier)" class="mw-redirect" title="ISSN (identifier)">ISSN</a> <a rel="nofollow" class="external text" href="https://search.worldcat.org/issn/1573-7659">1573-7659</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:21684923">21684923</a>.</cite>

</li>

<li id="cite_note-:1-14">^ <a href="#cite_ref-:1_14-0">a</a> <a href="#cite_ref-:1_14-1">b</a> <a href="#cite_ref-:1_14-2">c</a> <a href="#cite_ref-:1_14-3">d</a> <a href="#cite_ref-:1_14-4">e</a> <a href="#cite_ref-:1_14-5">f</a> <cite id="CITEREFChenGrangierAuli2016" class="citation journal cs1">Chen, Wenlin; Grangier, David; Auli, Michael (August 2016). <a rel="nofollow" class="external text" href="https://aclanthology.org/P16-1186">"Strategies for Training Large Vocabulary Neural Language Models"</a>. Proceedings of the 54th Annual Meeting of the Association for Computational Linguistics (Volume 1: Long Papers). Berlin, Germany: Association for Computational Linguistics: 1975–1985. <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1512.04906">1512.04906</a>. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.18653%2Fv1%2FP16-1186">10.18653/v1/P16-1186</a>. <a href="S2CID_(identifier)" class="mw-redirect" title="S2CID (identifier)">S2CID</a> <a rel="nofollow" class="external text" href="https://api.semanticscholar.org/CorpusID:6035643">6035643</a>.</cite>

</li>

<li id="cite_note-:2-15">^ <a href="#cite_ref-:2_15-0">a</a> <a href="#cite_ref-:2_15-1">b</a> <a href="#cite_ref-:2_15-2">c</a> <cite id="CITEREFMorinBengio2005" class="citation journal cs1">Morin, Frederic; Bengio, Yoshua (2005-01-06). <a rel="nofollow" class="external text" href="https://www.iro.umontreal.ca/~lisa/pointeurs/hierarchical-nnlm-aistats05.pdf">"Hierarchical Probabilistic Neural Network Language Model"</a> (PDF). International Workshop on Artificial Intelligence and Statistics. PMLR: 246–252.</cite>

</li>

<li id="cite_note-16"><a href="#cite_ref-16">^</a> <cite id="CITEREFMilakovGimelshein2018" class="citation arxiv cs1">Milakov, Maxim; Gimelshein, Natalia (2018). "Online normalizer calculation for softmax". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1805.02867">1805.02867</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/cs.PF">cs.PF</a>].</cite>

</li>

<li id="cite_note-17"><a href="#cite_ref-17">^</a> <cite id="CITEREFDaoFuErmonRudra2022" class="citation journal cs1">Dao, Tri; Fu, Dan; Ermon, Stefano; Rudra, Atri; Ré, Christopher (2022-12-06). <a rel="nofollow" class="external text" href="https://proceedings.neurips.cc/paper_files/paper/2022/hash/67d57c32e20fd0a7a302cb81d36e40d5-Abstract-Conference.html">"FlashAttention: Fast and Memory-Efficient Exact Attention with IO-Awareness"</a>. Advances in Neural Information Processing Systems. 35: 16344–16359.</cite>

</li>

<li id="cite_note-18"><a href="#cite_ref-18">^</a> <cite id="CITEREFBoltzmann1868" class="citation journal cs1"><a href="Ludwig_Boltzmann" title="Ludwig Boltzmann">Boltzmann, Ludwig</a> (1868). "Studien über das Gleichgewicht der lebendigen Kraft zwischen bewegten materiellen Punkten" [Studies on the balance of living force between moving material points]. Wiener Berichte. 58: 517–560.</cite>

</li>

<li id="cite_note-19"><a href="#cite_ref-19">^</a> <cite id="CITEREFGibbs1902" class="citation book cs1"><a href="Josiah_Willard_Gibbs" title="Josiah Willard Gibbs">Gibbs, Josiah Willard</a> (1902). <a href="Elementary_Principles_in_Statistical_Mechanics" title="Elementary Principles in Statistical Mechanics">Elementary Principles in Statistical Mechanics</a>.</cite>

</li>

<li id="cite_note-Gao-20">^ <a href="#cite_ref-Gao_20-0">a</a> <a href="#cite_ref-Gao_20-1">b</a> <cite id="CITEREFGaoPavel2017" class="citation arxiv cs1">Gao, Bolin; <a href="Lacra_Pavel" title="Lacra Pavel">Pavel, Lacra</a> (2017). "On the Properties of the Softmax Function with Application in Game Theory and Reinforcement Learning". <a href="ArXiv_(identifier)" class="mw-redirect" title="ArXiv (identifier)">arXiv</a>:<a rel="nofollow" class="external text" href="https://arxiv.org/abs/1704.00805">1704.00805</a> [<a rel="nofollow" class="external text" href="https://arxiv.org/archive/math.OC">math.OC</a>].</cite>

</li>

<li id="cite_note-21"><a href="#cite_ref-21">^</a> <cite id="CITEREFBridle1990a" class="citation conference cs1">Bridle, John S. (1990a). Soulié F.F.; Hérault J. (eds.). Probabilistic Interpretation of Feedforward Classification Network Outputs, with Relationships to Statistical Pattern Recognition. Neurocomputing: Algorithms, Architectures and Applications (1989). NATO ASI Series (Series F: Computer and Systems Sciences). Vol. 68. Berlin, Heidelberg: Springer. pp. 227–236. <a href="Doi_(identifier)" class="mw-redirect" title="Doi (identifier)">doi</a>:<a rel="nofollow" class="external text" href="https://doi.org/10.1007%2F978-3-642-76153-9_28">10.1007/978-3-642-76153-9_28</a>.</cite>

</li>

<li id="cite_note-22"><a href="#cite_ref-22">^</a> <cite id="CITEREFBridle1990b" class="citation conference cs1">Bridle, John S. (1990b). D. S. Touretzky (ed.). <a rel="nofollow" class="external text" href="https://papers.nips.cc/paper/195-training-stochastic-model-recognition-algorithms-as-networks-can-lead-to-maximum-mutual-information-estimation-of-parameters">Training Stochastic Model Recognition Algorithms as Networks can Lead to Maximum Mutual Information Estimation of Parameters</a>. <a href="Advances_in_Neural_Information_Processing_Systems" class="mw-redirect" title="Advances in Neural Information Processing Systems">Advances in Neural Information Processing Systems</a> 2 (1989). Morgan-Kaufmann.</cite>

</li>

<li id="cite_note-23"><a href="#cite_ref-23">^</a> "Speeding Up Entmax" by Maxat Tezekbayev, Vassilina Nikoulina, Matthias Gallé, Zhenisbek Assylbekov, <a rel="nofollow" class="external free" href="https://arxiv.org/abs/2111.06832v3">https://arxiv.org/abs/2111.06832v3</a>

</li>

</ol></div></div>

/* start https://en.wikipedia.org/ */

.mw-parser-output .navbox{box-sizing:border-box;border:1px solid #a2a9b1;width:100%;clear:both;font-size:88%;text-align:center;padding:1px;margin:1em auto 0}.mw-parser-output .navbox .navbox{margin-top:0}.mw-parser-output .navbox+.navbox,.mw-parser-output .navbox+.navbox-styles+.navbox{margin-top:-1px}.mw-parser-output .navbox-inner,.mw-parser-output .navbox-subgroup{width:100%}.mw-parser-output .navbox-group,.mw-parser-output .navbox-title,.mw-parser-output .navbox-abovebelow{padding:0.25em 1em;line-height:1.5em;text-align:center}.mw-parser-output .navbox-group{white-space:nowrap;text-align:right}.mw-parser-output .navbox,.mw-parser-output .navbox-subgroup{background-color:#fdfdfd}.mw-parser-output .navbox-list{line-height:1.5em;border-color:#fdfdfd}.mw-parser-output .navbox-list-with-group{text-align:left;border-left-width:2px;border-left-style:solid}.mw-parser-output tr+tr>.navbox-abovebelow,.mw-parser-output tr+tr>.navbox-group,.mw-parser-output tr+tr>.navbox-image,.mw-parser-output tr+tr>.navbox-list{border-top:2px solid #fdfdfd}.mw-parser-output .navbox-title{background-color:#ccf}.mw-parser-output .navbox-abovebelow,.mw-parser-output .navbox-group,.mw-parser-output .navbox-subgroup .navbox-title{background-color:#ddf}.mw-parser-output .navbox-subgroup .navbox-group,.mw-parser-output .navbox-subgroup .navbox-abovebelow{background-color:#e6e6ff}.mw-parser-output .navbox-even{background-color:#f7f7f7}.mw-parser-output .navbox-odd{background-color:transparent}.mw-parser-output .navbox .hlist td dl,.mw-parser-output .navbox .hlist td ol,.mw-parser-output .navbox .hlist td ul,.mw-parser-output .navbox td.hlist dl,.mw-parser-output .navbox td.hlist ol,.mw-parser-output .navbox td.hlist ul{padding:0.125em 0}.mw-parser-output .navbox .navbar{display:block;font-size:100%}.mw-parser-output .navbox-title .navbar{float:left;text-align:left;margin-right:0.5em}body.skin--responsive .mw-parser-output .navbox-image img{max-width:none!important}@media print{body.ns-0 .mw-parser-output .navbox{display:none!important}}

/* end https://en.wikipedia.org/ */

</style></div><div role="navigation" class="navbox" aria-labelledby="Artificial_intelligence_(AI)426" style="padding:3px"><table class="nowraplinks hlist mw-collapsible autocollapse navbox-inner" style="border-spacing:0;background:transparent;color:inherit"><tbody><tr><th scope="col" class="navbox-title" colspan="2"><div id="Artificial_intelligence_(AI)426" style="font-size:114%;margin:0 4em"><a href="Artificial_intelligence" title="Artificial intelligence">Artificial intelligence</a> (AI)</div></th></tr><tr><td class="navbox-abovebelow" colspan="2"><div>

<ul><li><a href="History_of_artificial_intelligence" title="History of artificial intelligence">History</a>

<ul><li><a href="Timeline_of_artificial_intelligence" title="Timeline of artificial intelligence">timeline</a></li></ul></li>

<li><a href="List_of_artificial_intelligence_companies" title="List of artificial intelligence companies">Companies</a></li>

<li><a href="List_of_artificial_intelligence_projects" title="List of artificial intelligence projects">Projects</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Concepts</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Parameter" title="Parameter">Parameter</a>

<ul><li><a href="Hyperparameter_(machine_learning)" title="Hyperparameter (machine learning)">Hyperparameter</a></li></ul></li>

<li><a href="Loss_functions_for_classification" title="Loss functions for classification">Loss functions</a></li>

<li><a href="Regression_analysis" title="Regression analysis">Regression</a>

<ul><li><a href="Bias%E2%80%93variance_tradeoff" title="Bias–variance tradeoff">Bias–variance tradeoff</a></li>

<li><a href="Double_descent" title="Double descent">Double descent</a></li>

<li><a href="Overfitting" title="Overfitting">Overfitting</a></li></ul></li>

<li><a href="Cluster_analysis" title="Cluster analysis">Clustering</a></li>

<li><a href="Gradient_descent" title="Gradient descent">Gradient descent</a>

<li><a href="Quasi-Newton_method" title="Quasi-Newton method">Quasi-Newton method</a></li>

<li><a href="Conjugate_gradient_method" title="Conjugate gradient method">Conjugate gradient method</a></li></ul></li>

<li><a href="Backpropagation" title="Backpropagation">Backpropagation</a></li>

<li><a href="Attention_(machine_learning)" title="Attention (machine learning)">Attention</a></li>

<li><a href="Convolution" title="Convolution">Convolution</a></li>

<li><a href="Normalization_(machine_learning)" title="Normalization (machine learning)">Normalization</a>

<ul><li><a href="Batch_normalization" title="Batch normalization">Batchnorm</a></li></ul></li>

<li><a href="Activation_function" title="Activation function">Activation</a>

<ul>

<li><a href="Sigmoid_function" title="Sigmoid function">Sigmoid</a></li>

<li><a href="Rectifier_(neural_networks)" title="Rectifier (neural networks)">Rectifier</a></li></ul></li>

<li><a href="Gating_mechanism" title="Gating mechanism">Gating</a></li>

<li><a href="Weight_initialization" title="Weight initialization">Weight initialization</a></li>

<li><a href="Regularization_(mathematics)" title="Regularization (mathematics)">Regularization</a></li>

<li><a href="Training%2C_validation%2C_and_test_data_sets" title="Training, validation, and test data sets">Datasets</a>

<ul><li><a href="Data_augmentation" title="Data augmentation">Augmentation</a></li></ul></li>

<li><a href="Prompt_engineering" title="Prompt engineering">Prompt engineering</a></li>

<li><a href="Reinforcement_learning" title="Reinforcement learning">Reinforcement learning</a>

<ul><li><a href="Q-learning" title="Q-learning">Q-learning</a></li>

<li><a href="State%E2%80%93action%E2%80%93reward%E2%80%93state%E2%80%93action" title="State–action–reward–state–action">SARSA</a></li>

<li><a href="Imitation_learning" title="Imitation learning">Imitation</a></li>

<li><a href="Policy_gradient_method" title="Policy gradient method">Policy gradient</a></li></ul></li>

<li><a href="Diffusion_process" title="Diffusion process">Diffusion</a></li>

<li><a href="Latent_diffusion_model" title="Latent diffusion model">Latent diffusion model</a></li>

<li><a href="Autoregressive_model" title="Autoregressive model">Autoregression</a></li>

<li><a href="Adversarial_machine_learning" title="Adversarial machine learning">Adversary</a></li>

<li><a href="Uncanny_valley" title="Uncanny valley">Uncanny valley</a></li>

<li><a href="Self-supervised_learning" title="Self-supervised learning">Self-supervised learning</a></li>

<li><a href="Reflection_(artificial_intelligence)" class="mw-redirect" title="Reflection (artificial intelligence)">Reflection</a></li>

<li><a href="Recursive_self-improvement" title="Recursive self-improvement">Recursive self-improvement</a></li>

<li><a href="Hallucination_(artificial_intelligence)" title="Hallucination (artificial intelligence)">Hallucination</a></li>

<li><a href="Word_embedding" title="Word embedding">Word embedding</a></li>

<li><a href="Vibe_coding" title="Vibe coding">Vibe coding</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Applications</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Machine_learning" title="Machine learning">Machine learning</a>

<ul><li><a href="Prompt_engineering#In-context_learning" title="Prompt engineering">In-context learning</a></li></ul></li>

<li><a href="Neural_network_(machine_learning)" title="Neural network (machine learning)">Artificial neural network</a>

<ul><li><a href="Deep_learning" title="Deep learning">Deep learning</a></li></ul></li>

<li><a href="Language_model" title="Language model">Language model</a>

<ul><li><a href="Large_language_model" title="Large language model">Large language model</a></li>

<li><a href="Reasoning_language_model" title="Reasoning language model">Reasoning language model</a></li>

<li><a href="Model_Context_Protocol" title="Model Context Protocol">Model Context Protocol</a></li>

<li><a href="Intelligent_agent" title="Intelligent agent">Intelligent agent</a></li>

<li><a href="Artificial_human_companion" title="Artificial human companion">Artificial human companion</a></li>

<li><a href="Humanity's_Last_Exam" title="Humanity's Last Exam">Humanity's Last Exam</a></li>

<li><a href="Artificial_general_intelligence" title="Artificial general intelligence">Artificial general intelligence (AGI)</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Implementations</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em"></div><table class="nowraplinks navbox-subgroup" style="border-spacing:0"><tbody><tr><th scope="row" class="navbox-group" style="width:1%">Audio–visual</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="AlexNet" title="AlexNet">AlexNet</a></li>

<li><a href="WaveNet" title="WaveNet">WaveNet</a></li>

<li><a href="Human_image_synthesis" title="Human image synthesis">Human image synthesis</a></li>

<li><a href="Computer_vision" title="Computer vision">Computer vision</a></li>

<li><a href="Deep_learning_speech_synthesis" title="Deep learning speech synthesis">Speech synthesis</a>

<li><a href="ElevenLabs" title="ElevenLabs">ElevenLabs</a></li></ul></li>

<li><a href="Speech_recognition" title="Speech recognition">Speech recognition</a>

<ul><li><a href="Whisper_(speech_recognition_system)" title="Whisper (speech recognition system)">Whisper</a></li></ul></li>

<li><a href="Facial_recognition_system" title="Facial recognition system">Facial recognition</a></li>

<li><a href="AlphaFold" title="AlphaFold">AlphaFold</a></li>

<li><a href="Text-to-image_model" title="Text-to-image model">Text-to-image models</a>

<ul><li><a href="Aurora_(text-to-image_model)" class="mw-redirect" title="Aurora (text-to-image model)">Aurora</a></li>

<li><a href="Adobe_Firefly" title="Adobe Firefly">Firefly</a></li>

<li><a href="Ideogram_(text-to-image_model)" title="Ideogram (text-to-image model)">Ideogram</a></li>

<li><a href="Imagen_(text-to-image_model)" title="Imagen (text-to-image model)">Imagen</a></li>

<li><a href="Midjourney" title="Midjourney">Midjourney</a></li>

<li><a href="Recraft" title="Recraft">Recraft</a></li>

<li><a href="Stable_Diffusion" title="Stable Diffusion">Stable Diffusion</a></li></ul></li>

<li><a href="Text-to-video_model" title="Text-to-video model">Text-to-video models</a>

<ul><li><a href="Dream_Machine_(text-to-video_model)" title="Dream Machine (text-to-video model)">Dream Machine</a></li>

<li><a href="Runway_(company)#Services_and_technologies" title="Runway (company)">Runway Gen</a></li>

<li><a href="MiniMax_(company)#Hailuo_AI" title="MiniMax (company)">Hailuo AI</a></li>

<li><a href="Kling_(text-to-video_model)" class="mw-redirect" title="Kling (text-to-video model)">Kling</a></li>

<li><a href="Music_and_artificial_intelligence" title="Music and artificial intelligence">Music generation</a>

<ul><li><a href="Riffusion" title="Riffusion">Riffusion</a></li>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Text</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em">

<li><a href="GloVe" title="GloVe">GloVe</a></li>

<li><a href="Llama_(language_model)" title="Llama (language model)">Llama</a></li>

<li><a href="Chinchilla_(language_model)" title="Chinchilla (language model)">Chinchilla AI</a></li>

<li><a href="ChatGPT" title="ChatGPT">ChatGPT</a></li>

<li><a href="Claude_(language_model)" title="Claude (language model)">Claude</a></li>

<li><a href="Gemini_(language_model)" title="Gemini (language model)">Gemini</a>

<ul><li><a href="Gemini_(chatbot)" title="Gemini (chatbot)">chatbot</a></li></ul></li>

<li><a href="LaMDA" title="LaMDA">LaMDA</a></li>

<li><a href="BLOOM_(language_model)" title="BLOOM (language model)">BLOOM</a></li>

<li><a href="Project_Debater" title="Project Debater">Project Debater</a></li>

<li><a href="IBM_Watson" title="IBM Watson">IBM Watson</a></li>

<li><a href="IBM_Watsonx" title="IBM Watsonx">IBM Watsonx</a></li>

<li><a href="IBM_Granite" title="IBM Granite">Granite</a></li>

<li><a href="Huawei_PanGu" title="Huawei PanGu">PanGu-Σ</a></li>

<li><a href="DeepSeek_(chatbot)" title="DeepSeek (chatbot)">DeepSeek</a></li>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Decisional</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="AlphaGo" title="AlphaGo">AlphaGo</a></li>

<li><a href="AlphaZero" title="AlphaZero">AlphaZero</a></li>

<li><a href="OpenAI_Five" title="OpenAI Five">OpenAI Five</a></li>

<li><a href="Self-driving_car" title="Self-driving car">Self-driving car</a></li>

<li><a href="MuZero" title="MuZero">MuZero</a></li>

<li><a href="Action_selection" title="Action selection">Action selection</a>

<ul><li><a href="AutoGPT" title="AutoGPT">AutoGPT</a></li></ul></li>

<li><a href="Robot_control" title="Robot control">Robot control</a></li></ul>

</div></td></tr></tbody></table><div></div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">People</th><td class="navbox-list-with-group navbox-list navbox-even" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Alan_Turing" title="Alan Turing">Alan Turing</a></li>

<li><a href="Warren_Sturgis_McCulloch" title="Warren Sturgis McCulloch">Warren Sturgis McCulloch</a></li>

<li><a href="Walter_Pitts" title="Walter Pitts">Walter Pitts</a></li>

<li><a href="John_von_Neumann" title="John von Neumann">John von Neumann</a></li>

<li><a href="Claude_Shannon" title="Claude Shannon">Claude Shannon</a></li>

<li><a href="Shun'ichi_Amari" title="Shun'ichi Amari">Shun'ichi Amari</a></li>

<li><a href="Kunihiko_Fukushima" title="Kunihiko Fukushima">Kunihiko Fukushima</a></li>

<li><a href="Takeo_Kanade" title="Takeo Kanade">Takeo Kanade</a></li>

<li><a href="Marvin_Minsky" title="Marvin Minsky">Marvin Minsky</a></li>

<li><a href="John_McCarthy_(computer_scientist)" title="John McCarthy (computer scientist)">John McCarthy</a></li>

<li><a href="Nathaniel_Rochester_(computer_scientist)" title="Nathaniel Rochester (computer scientist)">Nathaniel Rochester</a></li>

<li><a href="Allen_Newell" title="Allen Newell">Allen Newell</a></li>

<li><a href="Cliff_Shaw" title="Cliff Shaw">Cliff Shaw</a></li>

<li><a href="Herbert_A._Simon" title="Herbert A. Simon">Herbert A. Simon</a></li>

<li><a href="Oliver_Selfridge" title="Oliver Selfridge">Oliver Selfridge</a></li>

<li><a href="Frank_Rosenblatt" title="Frank Rosenblatt">Frank Rosenblatt</a></li>

<li><a href="Bernard_Widrow" title="Bernard Widrow">Bernard Widrow</a></li>

<li><a href="Joseph_Weizenbaum" title="Joseph Weizenbaum">Joseph Weizenbaum</a></li>

<li><a href="Seymour_Papert" title="Seymour Papert">Seymour Papert</a></li>

<li><a href="Seppo_Linnainmaa" title="Seppo Linnainmaa">Seppo Linnainmaa</a></li>

<li><a href="Paul_Werbos" title="Paul Werbos">Paul Werbos</a></li>

<li><a href="Geoffrey_Hinton" title="Geoffrey Hinton">Geoffrey Hinton</a></li>

<li><a href="John_Hopfield" title="John Hopfield">John Hopfield</a></li>

<li><a href="J%C3%BCrgen_Schmidhuber" title="Jürgen Schmidhuber">Jürgen Schmidhuber</a></li>

<li><a href="Yann_LeCun" title="Yann LeCun">Yann LeCun</a></li>

<li><a href="Yoshua_Bengio" title="Yoshua Bengio">Yoshua Bengio</a></li>

<li><a href="Lotfi_A._Zadeh" title="Lotfi A. Zadeh">Lotfi A. Zadeh</a></li>

<li><a href="Stephen_Grossberg" title="Stephen Grossberg">Stephen Grossberg</a></li>

<li><a href="Alex_Graves_(computer_scientist)" title="Alex Graves (computer scientist)">Alex Graves</a></li>

<li><a href="James_Goodnight" title="James Goodnight">James Goodnight</a></li>

<li><a href="Andrew_Ng" title="Andrew Ng">Andrew Ng</a></li>

<li><a href="Ilya_Sutskever" title="Ilya Sutskever">Ilya Sutskever</a></li>

<li><a href="Alex_Krizhevsky" title="Alex Krizhevsky">Alex Krizhevsky</a></li>

<li><a href="Ian_Goodfellow" title="Ian Goodfellow">Ian Goodfellow</a></li>

<li><a href="Demis_Hassabis" title="Demis Hassabis">Demis Hassabis</a></li>

<li><a href="David_Silver_(computer_scientist)" title="David Silver (computer scientist)">David Silver</a></li>

<li><a href="Andrej_Karpathy" title="Andrej Karpathy">Andrej Karpathy</a></li>

<li><a href="Ashish_Vaswani" title="Ashish Vaswani">Ashish Vaswani</a></li>

<li><a href="Noam_Shazeer" title="Noam Shazeer">Noam Shazeer</a></li>

<li><a href="Aidan_Gomez" title="Aidan Gomez">Aidan Gomez</a></li>

<li><a href="Mustafa_Suleyman" title="Mustafa Suleyman">Mustafa Suleyman</a></li>

<li><a href="Fran%C3%A7ois_Chollet" title="François Chollet">François Chollet</a></li></ul>

</div></td></tr><tr><th scope="row" class="navbox-group" style="width:1%">Architectures</th><td class="navbox-list-with-group navbox-list navbox-odd" style="width:100%;padding:0"><div style="padding:0 0.25em">

<ul><li><a href="Neural_Turing_machine" title="Neural Turing machine">Neural Turing machine</a></li>

<li><a href="Differentiable_neural_computer" title="Differentiable neural computer">Differentiable neural computer</a></li>

<li><a href="Transformer_(deep_learning_architecture)" title="Transformer (deep learning architecture)">Transformer</a>

<ul><li><a href="Vision_transformer" title="Vision transformer">Vision transformer (ViT)</a></li></ul></li>

<li><a href="Recurrent_neural_network" title="Recurrent neural network">Recurrent neural network (RNN)</a></li>

<li><a href="Long_short-term_memory" title="Long short-term memory">Long short-term memory (LSTM)</a></li>

<li><a href="Gated_recurrent_unit" title="Gated recurrent unit">Gated recurrent unit (GRU)</a></li>

<li><a href="Echo_state_network" title="Echo state network">Echo state network</a></li>

<li><a href="Multilayer_perceptron" title="Multilayer perceptron">Multilayer perceptron (MLP)</a></li>

<li><a href="Convolutional_neural_network" title="Convolutional neural network">Convolutional neural network (CNN)</a></li>

<li><a href="Residual_neural_network" title="Residual neural network">Residual neural network (RNN)</a></li>

<li><a href="Highway_network" title="Highway network">Highway network</a></li>

<li><a href="Mamba_(deep_learning_architecture)" title="Mamba (deep learning architecture)">Mamba</a></li>

<li><a href="Autoencoder" title="Autoencoder">Autoencoder</a></li>

<li><a href="Variational_autoencoder" title="Variational autoencoder">Variational autoencoder (VAE)</a></li>

<li><a href="Generative_adversarial_network" title="Generative adversarial network">Generative adversarial network (GAN)</a></li>

<li><a href="Graph_neural_network" title="Graph neural network">Graph neural network (GNN)</a></li></ul>

</div></td></tr><tr><td class="navbox-abovebelow" colspan="2"><div>

<ul><li> Category</li></ul>

</div></td></tr></tbody></table></div></div><div><div class="zim-footer">

This article is issued from <a class="external text" title="Last edited on 2025-05-29" href="https://en.wikipedia.org/wiki/?title=Softmax_function&oldid=1292950500">Wikipedia</a>. The text is available under <a class="external text" href="https://creativecommons.org/licenses/by-sa/4.0/deed.en">Creative Commons Attribution-Share Alike 4.0</a> unless otherwise noted. Additional terms may apply for the media files.

</div>

</div></div>

</div>

</main>

</div>

</body></html>